diff options
Diffstat (limited to 'yjit/src')
| -rw-r--r-- | yjit/src/asm/mod.rs | 4 | ||||
| -rw-r--r-- | yjit/src/backend/arm64/mod.rs | 17 | ||||
| -rw-r--r-- | yjit/src/backend/ir.rs | 16 | ||||
| -rw-r--r-- | yjit/src/backend/tests.rs | 4 | ||||
| -rw-r--r-- | yjit/src/codegen.rs | 608 | ||||
| -rw-r--r-- | yjit/src/core.rs | 133 | ||||
| -rw-r--r-- | yjit/src/cruby.rs | 34 | ||||
| -rw-r--r-- | yjit/src/cruby_bindings.inc.rs | 588 | ||||
| -rw-r--r-- | yjit/src/invariants.rs | 10 | ||||
| -rw-r--r-- | yjit/src/stats.rs | 74 | ||||
| -rw-r--r-- | yjit/src/utils.rs | 15 | ||||
| -rw-r--r-- | yjit/src/virtualmem.rs | 13 | ||||
| -rw-r--r-- | yjit/src/yjit.rs | 18 |
13 files changed, 770 insertions, 764 deletions
diff --git a/yjit/src/asm/mod.rs b/yjit/src/asm/mod.rs index ebdc205da9..9ef675b34d 100644 --- a/yjit/src/asm/mod.rs +++ b/yjit/src/asm/mod.rs @@ -441,12 +441,12 @@ impl CodeBlock { // Ignore empty code ranges if start_addr == end_addr { - return (0..0).into_iter(); + return 0..0; } let start_page = (start_addr.raw_addr(self) - mem_start) / self.page_size; let end_page = (end_addr.raw_addr(self) - mem_start - 1) / self.page_size; - (start_page..end_page + 1).into_iter() + start_page..end_page + 1 } /// Get a (possibly dangling) direct pointer to the current write position diff --git a/yjit/src/backend/arm64/mod.rs b/yjit/src/backend/arm64/mod.rs index 66e333f867..1712ad0302 100644 --- a/yjit/src/backend/arm64/mod.rs +++ b/yjit/src/backend/arm64/mod.rs @@ -98,7 +98,7 @@ fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) { #[cfg(not(test))] { let end = cb.get_write_ptr(); - unsafe { rb_yjit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; } } @@ -878,14 +878,13 @@ impl Assembler } } - /// Emit a push instruction for the given operand by adding to the stack - /// pointer and then storing the given value. + /// Push a value to the stack by subtracting from the stack pointer then storing, + /// leaving an 8-byte gap for alignment. fn emit_push(cb: &mut CodeBlock, opnd: A64Opnd) { str_pre(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP)); } - /// Emit a pop instruction into the given operand by loading the value - /// and then subtracting from the stack pointer. + /// Pop a value from the stack by loading `[sp]` then adding to the stack pointer. fn emit_pop(cb: &mut CodeBlock, opnd: A64Opnd) { ldr_post(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); } @@ -1155,8 +1154,8 @@ impl Assembler let regs = Assembler::get_caller_save_regs(); // Pop the state/flags register - msr(cb, SystemRegister::NZCV, Self::SCRATCH0); emit_pop(cb, Self::SCRATCH0); + msr(cb, SystemRegister::NZCV, Self::SCRATCH0); for reg in regs.into_iter().rev() { emit_pop(cb, A64Opnd::Reg(reg)); @@ -1361,7 +1360,7 @@ impl Assembler #[cfg(not(test))] cb.without_page_end_reserve(|cb| { for (start, end) in cb.writable_addrs(start_ptr, cb.get_write_ptr()) { - unsafe { rb_yjit_icache_invalidate(start as _, end as _) }; + unsafe { rb_jit_icache_invalidate(start as _, end as _) }; } }); @@ -1419,7 +1418,7 @@ mod tests { fn test_emit_cpop_all() { let (mut asm, mut cb) = setup_asm(); - asm.cpop_all(); + asm.cpop_all(crate::core::RegMapping::default()); asm.compile_with_num_regs(&mut cb, 0); } @@ -1778,7 +1777,7 @@ mod tests { assert_disasm!(cb, "e1ff9fd2e10370b2", {" 0x0: mov x1, #0xffff - 0x4: orr x1, xzr, #0x10000 + 0x4: mov x1, #0x10000 "}); } diff --git a/yjit/src/backend/ir.rs b/yjit/src/backend/ir.rs index b704a24985..e14f3f9cb2 100644 --- a/yjit/src/backend/ir.rs +++ b/yjit/src/backend/ir.rs @@ -6,7 +6,7 @@ use crate::codegen::{gen_counted_exit, gen_outlined_exit}; use crate::cruby::{vm_stack_canary, SIZEOF_VALUE_I32, VALUE, VM_ENV_DATA_SIZE}; use crate::virtualmem::CodePtr; use crate::asm::{CodeBlock, OutlinedCb}; -use crate::core::{Context, RegMapping, RegOpnd, MAX_CTX_TEMPS}; +use crate::core::{Context, RegMapping, RegOpnd, MAX_CTX_LOCALS, MAX_CTX_TEMPS}; use crate::options::*; use crate::stats::*; @@ -242,7 +242,9 @@ impl Opnd let last_idx = stack_size as i32 + VM_ENV_DATA_SIZE as i32 - 1; assert!(last_idx <= idx, "Local index {} must be >= last local index {}", idx, last_idx); assert!(idx <= last_idx + num_locals as i32, "Local index {} must be < last local index {} + local size {}", idx, last_idx, num_locals); - RegOpnd::Local((last_idx + num_locals as i32 - idx) as u8) + // Indices that don't fit in u8 are capped to MAX_CTX_LOCALS, which is untrackable. + let local_idx = last_idx + num_locals as i32 - idx; + RegOpnd::Local(local_idx.try_into().unwrap_or(MAX_CTX_LOCALS as u8)) } else { assert!(idx < stack_size as i32); RegOpnd::Stack((stack_size as i32 - idx - 1) as u8) @@ -1602,7 +1604,7 @@ impl Assembler if c_args.len() > 0 { // Resolve C argument dependencies let c_args_len = c_args.len() as isize; - let moves = Self::reorder_reg_moves(&c_args.drain(..).into_iter().collect()); + let moves = Self::reorder_reg_moves(&std::mem::take(&mut c_args)); shift_live_ranges(&mut shifted_live_ranges, asm.insns.len(), moves.len() as isize - c_args_len); // Push batched C arguments @@ -1824,12 +1826,12 @@ impl Assembler { out } - pub fn cpop_all(&mut self) { + pub fn cpop_all(&mut self, reg_mapping: RegMapping) { self.push_insn(Insn::CPopAll); // Re-enable ccall's RegMappings assertion disabled by cpush_all. // cpush_all + cpop_all preserve all stack temp registers, so it's safe. - self.set_reg_mapping(self.ctx.get_reg_mapping()); + self.set_reg_mapping(reg_mapping); } pub fn cpop_into(&mut self, opnd: Opnd) { @@ -1840,14 +1842,16 @@ impl Assembler { self.push_insn(Insn::CPush(opnd)); } - pub fn cpush_all(&mut self) { + pub fn cpush_all(&mut self) -> RegMapping { self.push_insn(Insn::CPushAll); // Mark all temps as not being in registers. // Temps will be marked back as being in registers by cpop_all. // We assume that cpush_all + cpop_all are used for C functions in utils.rs // that don't require spill_regs for GC. + let mapping = self.ctx.get_reg_mapping(); self.set_reg_mapping(RegMapping::default()); + mapping } pub fn cret(&mut self, opnd: Opnd) { diff --git a/yjit/src/backend/tests.rs b/yjit/src/backend/tests.rs index ac2f35b3d9..bfeea5163a 100644 --- a/yjit/src/backend/tests.rs +++ b/yjit/src/backend/tests.rs @@ -232,9 +232,9 @@ fn test_jcc_ptr() let (mut asm, mut cb) = setup_asm(); let side_exit = Target::CodePtr(cb.get_write_ptr().add_bytes(4)); - let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK)); + let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK as i32)); asm.test( - Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG), + Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32), not_mask, ); asm.jnz(side_exit); diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 9c06010527..743a10e15a 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -234,20 +234,36 @@ impl<'a> JITState<'a> { result } - /// Return true if the current ISEQ could escape an environment. + /// Return true if the JIT code can use [`Self::assume_no_ep_escape`] + /// and will run with an on-stack (`!VM_ENV_ESCAPED_P`) environment. /// - /// As of vm_push_frame(), EP is always equal to BP. However, after pushing - /// a frame, some ISEQ setups call vm_bind_update_env(), which redirects EP. - /// Also, some method calls escape the environment to the heap. - fn escapes_ep(&self) -> bool { + /// ## Reasoning about ISEQs that are not currently running + /// + /// As of vm_push_frame() and its JIT code equivalent, EP is always equal to BP (the + /// environment is on-stack and has not escaped). We can usually assume this is the starting + /// condition upon entry into JIT code. However, after pushing a frame and before entry into + /// JIT code, some ISEQ setups call vm_bind_update_env(), which redirects EP. + /// + /// ## After making the assumption + /// + /// After JIT code entry, many ruby operations can have the environment escape to heap. These + /// are handled by [`crate::invariants`]. + /// + /// Exceptional entry through jit_exec_exception() is an extreme case of the environment state + /// changing between vm_push_frame() and entry into JIT code. The frame could have been pushed + /// before YJIT is enabled. The exception entry point refuses entry with an escaped environment. + fn can_assume_on_stack_env(&self) -> bool { match unsafe { get_iseq_body_type(self.iseq) } { // <main> frame is always associated to TOPLEVEL_BINDING. ISEQ_TYPE_MAIN | // Kernel#eval uses a heap EP when a Binding argument is not nil. - ISEQ_TYPE_EVAL => true, - // If this ISEQ has previously escaped EP, give up the optimization. - _ if iseq_escapes_ep(self.iseq) => true, - _ => false, + ISEQ_TYPE_EVAL => false, + // Check the running environment if compiling for it + _ if unsafe { self.iseq == get_cfp_iseq(self.get_cfp()) && cfp_env_has_escaped(self.get_cfp()) } => false, + // If we've seen this ISEQ run with an escaped environment, give up the optimization + // to avoid excessive invalidations (even though it may be fine for soundness). + _ if seen_escaped_env(self.iseq) => false, + _ => true, } } @@ -376,8 +392,8 @@ impl<'a> JITState<'a> { if jit_ensure_block_entry_exit(self, asm).is_none() { return false; // out of space, give up } - if self.escapes_ep() { - return false; // EP has been escaped in this ISEQ. disable the optimization to avoid an invalidation loop. + if !self.can_assume_on_stack_env() { + return false; // Unsound or unprofitable to make the assumption } self.no_ep_escape = true; true @@ -438,7 +454,7 @@ impl<'a> JITState<'a> { fn flush_perf_symbols(&self, cb: &CodeBlock) { assert_eq!(0, self.perf_stack.len()); let path = format!("/tmp/perf-{}.map", std::process::id()); - let mut f = std::fs::File::options().create(true).append(true).open(path).unwrap(); + let mut f = std::io::BufWriter::new(std::fs::File::options().create(true).append(true).open(path).unwrap()); for sym in self.perf_map.borrow().iter() { if let (start, Some(end), name) = sym { // In case the code straddles two pages, part of it belongs to the symbol. @@ -821,11 +837,11 @@ fn gen_stub_exit(ocb: &mut OutlinedCb) -> Option<CodePtr> { /// Generate an exit to return to the interpreter fn gen_exit(exit_pc: *mut VALUE, asm: &mut Assembler) { - #[cfg(all(feature = "disasm", not(test)))] - { + #[cfg(not(test))] + asm_comment!(asm, "exit to interpreter on {}", { let opcode = unsafe { rb_vm_insn_addr2opcode((*exit_pc).as_ptr()) }; - asm_comment!(asm, "exit to interpreter on {}", insn_name(opcode as usize)); - } + insn_name(opcode as usize) + }); if asm.ctx.is_return_landing() { asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); @@ -1094,11 +1110,7 @@ pub fn gen_entry_prologue( let code_ptr = cb.get_write_ptr(); let mut asm = Assembler::new(unsafe { get_iseq_body_local_table_size(iseq) }); - if get_option_ref!(dump_disasm).is_some() { - asm_comment!(asm, "YJIT entry point: {}", iseq_get_location(iseq, 0)); - } else { - asm_comment!(asm, "YJIT entry"); - } + asm_comment!(asm, "YJIT entry point: {}", iseq_get_location(iseq, 0)); asm.frame_setup(); @@ -1212,7 +1224,7 @@ fn gen_check_ints( // Not checking interrupt_mask since it's zero outside finalize_deferred_heap_pages, // signal_exec, or rb_postponed_job_flush. - let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG)); + let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32)); asm.test(interrupt_flag, interrupt_flag); asm.jnz(Target::side_exit(counter)); @@ -1296,7 +1308,6 @@ pub fn gen_single_block( let mut asm = Assembler::new(jit.num_locals()); asm.ctx = ctx; - #[cfg(feature = "disasm")] if get_option_ref!(dump_disasm).is_some() { let blockid_idx = blockid.idx; let chain_depth = if asm.ctx.get_chain_depth() > 0 { format!("(chain_depth: {})", asm.ctx.get_chain_depth()) } else { "".to_string() }; @@ -2263,7 +2274,8 @@ fn gen_expandarray( let comptime_recv = jit.peek_at_stack(&asm.ctx, 0); - // If the comptime receiver is not an array + // If the comptime receiver is not an array, speculate for when the `rb_check_array_type()` + // conversion returns nil and without side-effects (e.g. arbitrary method calls). if !unsafe { RB_TYPE_P(comptime_recv, RUBY_T_ARRAY) } { // at compile time, ensure to_ary is not defined let target_cme = unsafe { rb_callable_method_entry_or_negative(comptime_recv.class_of(), ID!(to_ary)) }; @@ -2275,13 +2287,19 @@ fn gen_expandarray( return None; } + // Bail when method_missing is defined to avoid generating code to call it. + // Also, for simplicity, bail when BasicObject#method_missing has been removed. + if !assume_method_basic_definition(jit, asm, comptime_recv.class_of(), ID!(method_missing)) { + gen_counter_incr(jit, asm, Counter::expandarray_method_missing); + return None; + } + // invalidate compile block if to_ary is later defined jit.assume_method_lookup_stable(asm, target_cme); jit_guard_known_klass( jit, asm, - comptime_recv.class_of(), array_opnd, array_opnd.into(), comptime_recv, @@ -2311,7 +2329,7 @@ fn gen_expandarray( } // Get the compile-time array length - let comptime_len = unsafe { rb_yjit_array_len(comptime_recv) as u32 }; + let comptime_len = unsafe { rb_jit_array_len(comptime_recv) as u32 }; // Move the array from the stack and check that it's an array. guard_object_is_array( @@ -2423,22 +2441,83 @@ fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { ep_opnd } -// Gets the EP of the ISeq of the containing method, or "local level". -// Equivalent of GET_LEP() macro. +/// Get the EP of the ISeq of the containing method, or "local level EP". +/// Equivalent to `GET_LEP()` with a constraint: +/// `ISEQ_TYPE_METHOD == iseq_under_compilation->body->local_iseq->body->type`. +/// No bad memory access happens when this condition is not met, just that the +/// EP returned may not be `VM_ENV_FLAG_LOCAL`. Practically, ISeqs that don't +/// meet this condition, such as `ISEQ_TYPE_TOP`, also don't need to use this operation. fn gen_get_lep(jit: &JITState, asm: &mut Assembler) -> Opnd { - // Equivalent of get_lvar_level() in compile.c - fn get_lvar_level(iseq: IseqPtr) -> u32 { - if iseq == unsafe { rb_get_iseq_body_local_iseq(iseq) } { - 0 - } else { - 1 + get_lvar_level(unsafe { rb_get_iseq_body_parent_iseq(iseq) }) - } - } - - let level = get_lvar_level(jit.get_iseq()); + // GET_LEP() chases the parent environment pointer to reach the local environment. Inside + // a descendant of ISEQ_TYPE_METHOD, it chases the same number of times as chasing + // the parent iseq pointer to reach the local iseq. See get_lvar_level() in compile.c + let mut iseq = jit.get_iseq(); + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(iseq) }; + let mut level = 0; + while iseq != local_iseq { + iseq = unsafe { rb_get_iseq_body_parent_iseq(iseq) }; + level += 1; + } + asm_comment!(asm, "get_lep(level: {level})"); gen_get_ep(asm, level) } +/// Load the value in the ME_CREF slot of the EP that contains the running CME. +/// Returns the slot value directly, which is only useful for guarding that the +/// CME has not changed. Unlike rb_vm_frame_method_entry(), we never dereference +/// `ep[VM_ENV_DATA_INDEX_ME_CREF]` but rather rely on `ep[VM_ENV_DATA_INDEX_FLAGS]` +/// to terminate the search, since we load the flags anyways for EP hopping. +/// When `ep[VM_ENV_DATA_INDEX_ME_CREF]` is not a CME, it can't match the expected +/// CME, and the guard fails. +fn gen_get_running_cme_or_sentinal(jit: &JITState, asm: &mut Assembler) -> Opnd { + // When not in a block, the running CME is at `cfp->ep`. + if jit.iseq == unsafe { rb_get_iseq_body_local_iseq(jit.iseq) } { + asm_comment!(asm, "cfp->ep[VM_ENV_DATA_INDEX_ME_CREF]"); + let lep_opnd = gen_get_ep(asm, 0); + Opnd::mem( + VALUE_BITS, + lep_opnd, + SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF, + ) + } else { + // Loop through EPs to find the one containing the CME. + // Stop when we find an EP with VM_FRAME_FLAG_BMETHOD (bmethod frame with CME) + // or VM_ENV_FLAG_LOCAL (local frame which is METHOD/CFUNC/IFUNC with CME). + // + // We cannot unroll to a static hop count like gen_get_lep() because bmethods + // defined inside methods may have a CME that lives at a EP cloers to the + // starting EP than at the local and final EP level. Each level of nesting can + // dynamically run with and without VM_FRAME_FLAG_BMETHOD set. + asm_comment!(asm, "search for running cme"); + let loop_label = asm.new_label("cme_loop"); + let done_label = asm.new_label("cme_done"); + + let ep_opnd = Opnd::mem(VALUE_BITS, CFP, RUBY_OFFSET_CFP_EP); + let ep_opnd = asm.load(ep_opnd); + + asm.write_label(loop_label); + // Load flags from ep[VM_ENV_DATA_INDEX_FLAGS] + let flags = asm.load(Opnd::mem(VALUE_BITS, ep_opnd, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32))); + // Check if VM_FRAME_FLAG_BMETHOD or VM_ENV_FLAG_LOCAL is set. + // If either is set, this EP contains the CME. + let check_flags = (VM_FRAME_FLAG_BMETHOD | VM_ENV_FLAG_LOCAL).as_usize(); + asm.test(flags, check_flags.into()); + asm.jnz(done_label); + // Get the previous EP from the current EP + // See GET_PREV_EP(ep) macro + // VALUE *prev_ep = ((VALUE *)((ep)[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03)) + let offs = SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL; + let next_ep_opnd = asm.load(Opnd::mem(VALUE_BITS, ep_opnd, offs)); + let next_ep_opnd = asm.and(next_ep_opnd, (!0x03_i64).into()); + asm.load_into(ep_opnd, next_ep_opnd); + asm.jmp(loop_label); + asm.write_label(done_label); + + // Load and return the CME from ep[VM_ENV_DATA_INDEX_ME_CREF] + asm.load(Opnd::mem(VALUE_BITS, ep_opnd, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF)) + } +} + fn gen_getlocal_generic( jit: &mut JITState, asm: &mut Assembler, @@ -2446,7 +2525,7 @@ fn gen_getlocal_generic( level: u32, ) -> Option<CodegenStatus> { // Split the block if we need to invalidate this instruction when EP escapes - if level == 0 && !jit.escapes_ep() && !jit.at_compile_target() { + if level == 0 && jit.can_assume_on_stack_env() && !jit.at_compile_target() { return jit.defer_compilation(asm); } @@ -2516,6 +2595,7 @@ fn gen_setlocal_generic( ep_offset: u32, level: u32, ) -> Option<CodegenStatus> { + // Post condition: The type of of the set local is updated in the Context. let value_type = asm.ctx.get_opnd_type(StackOpnd(0)); // Fallback because of write barrier @@ -2537,11 +2617,16 @@ fn gen_setlocal_generic( ); asm.stack_pop(1); + // Set local type in the context + if level == 0 { + let local_idx = ep_offset_to_local_idx(jit.get_iseq(), ep_offset).as_usize(); + asm.ctx.set_local_type(local_idx, value_type); + } return Some(KeepCompiling); } // Split the block if we need to invalidate this instruction when EP escapes - if level == 0 && !jit.escapes_ep() && !jit.at_compile_target() { + if level == 0 && jit.can_assume_on_stack_env() && !jit.at_compile_target() { return jit.defer_compilation(asm); } @@ -2589,6 +2674,7 @@ fn gen_setlocal_generic( ); } + // Set local type in the context if level == 0 { let local_idx = ep_offset_to_local_idx(jit.get_iseq(), ep_offset).as_usize(); asm.ctx.set_local_type(local_idx, value_type); @@ -2678,7 +2764,7 @@ fn gen_newhash( Some(KeepCompiling) } -fn gen_putstring( +fn gen_dupstring( jit: &mut JITState, asm: &mut Assembler, ) -> Option<CodegenStatus> { @@ -2698,7 +2784,7 @@ fn gen_putstring( Some(KeepCompiling) } -fn gen_putchilledstring( +fn gen_dupchilledstring( jit: &mut JITState, asm: &mut Assembler, ) -> Option<CodegenStatus> { @@ -2753,7 +2839,7 @@ fn gen_checkkeyword( ) -> Option<CodegenStatus> { // When a keyword is unspecified past index 32, a hash will be used // instead. This can only happen in iseqs taking more than 32 keywords. - if unsafe { (*get_iseq_body_param_keyword(jit.iseq)).num >= 32 } { + if unsafe { (*get_iseq_body_param_keyword(jit.iseq)).num >= VM_KW_SPECIFIED_BITS_MAX.try_into().unwrap() } { return None; } @@ -2848,24 +2934,12 @@ fn gen_get_ivar( recv: Opnd, recv_opnd: YARVOpnd, ) -> Option<CodegenStatus> { - let comptime_val_klass = comptime_receiver.class_of(); - // If recv isn't already a register, load it. let recv = match recv { Opnd::InsnOut { .. } => recv, _ => asm.load(recv), }; - // Check if the comptime class uses a custom allocator - let custom_allocator = unsafe { rb_get_alloc_func(comptime_val_klass) }; - let uses_custom_allocator = match custom_allocator { - Some(alloc_fun) => { - let allocate_instance = rb_class_allocate_instance as *const u8; - alloc_fun as *const u8 != allocate_instance - } - None => false, - }; - // Check if the comptime receiver is a T_OBJECT let receiver_t_object = unsafe { RB_TYPE_P(comptime_receiver, RUBY_T_OBJECT) }; // Use a general C call at the last chain to avoid exits on megamorphic shapes @@ -2874,12 +2948,9 @@ fn gen_get_ivar( gen_counter_incr(jit, asm, Counter::num_getivar_megamorphic); } - // If the class uses the default allocator, instances should all be T_OBJECT - // NOTE: This assumes nobody changes the allocator of the class after allocation. - // Eventually, we can encode whether an object is T_OBJECT or not - // inside object shapes. + // NOTE: This assumes T_OBJECT can't ever have the same shape_id as any other type. // too-complex shapes can't use index access, so we use rb_ivar_get for them too. - if !receiver_t_object || uses_custom_allocator || comptime_receiver.shape_too_complex() || megamorphic { + if !comptime_receiver.heap_object_p() || comptime_receiver.shape_complex() || megamorphic { // General case. Call rb_ivar_get(). // VALUE rb_ivar_get(VALUE obj, ID id) asm_comment!(asm, "call rb_ivar_get()"); @@ -2904,7 +2975,7 @@ fn gen_get_ivar( let ivar_index = unsafe { let shape_id = comptime_receiver.shape_id_of(); - let mut ivar_index: u16 = 0; + let mut ivar_index: attr_index_t = 0; if rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) { Some(ivar_index as usize) } else { @@ -2915,9 +2986,6 @@ fn gen_get_ivar( // Guard heap object (recv_opnd must be used before stack_pop) guard_object_is_heap(asm, recv, recv_opnd, Counter::getivar_not_heap); - // Compile time self is embedded and the ivar index lands within the object - let embed_test_result = unsafe { FL_TEST_RAW(comptime_receiver, VALUE(ROBJECT_EMBED.as_usize())) != VALUE(0) }; - let expected_shape = unsafe { rb_obj_shape_id(comptime_receiver) }; let shape_id_offset = unsafe { rb_shape_id_offset() }; let shape_opnd = Opnd::mem(SHAPE_ID_NUM_BITS as u8, recv, shape_id_offset); @@ -2946,28 +3014,37 @@ fn gen_get_ivar( asm.mov(out_opnd, Qnil.into()); } Some(ivar_index) => { - if embed_test_result { - // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h - - // Load the variable - let offs = ROBJECT_OFFSET_AS_ARY as i32 + (ivar_index * SIZEOF_VALUE) as i32; - let ivar_opnd = Opnd::mem(64, recv, offs); - - // Push the ivar on the stack - let out_opnd = asm.stack_push(Type::Unknown); - asm.mov(out_opnd, ivar_opnd); + let ivar_opnd = if receiver_t_object { + if comptime_receiver.embedded_p() { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + + // Load the variable + let offs = ROBJECT_OFFSET_AS_ARY as i32 + (ivar_index * SIZEOF_VALUE) as i32; + Opnd::mem(64, recv, offs) + } else { + // Compile time value is *not* embedded. + + // Get a pointer to the extended table + let tbl_opnd = asm.load(Opnd::mem(64, recv, ROBJECT_OFFSET_AS_HEAP_FIELDS as i32)); + + // Read the ivar from the extended table + Opnd::mem(64, tbl_opnd, (SIZEOF_VALUE * ivar_index) as i32) + } } else { - // Compile time value is *not* embedded. - - // Get a pointer to the extended table - let tbl_opnd = asm.load(Opnd::mem(64, recv, ROBJECT_OFFSET_AS_HEAP_FIELDS as i32)); + asm_comment!(asm, "call rb_ivar_get_at()"); - // Read the ivar from the extended table - let ivar_opnd = Opnd::mem(64, tbl_opnd, (SIZEOF_VALUE * ivar_index) as i32); + if assume_single_ractor_mode(jit, asm) { + asm.ccall(rb_ivar_get_at_no_ractor_check as *const u8, vec![recv, Opnd::UImm((ivar_index as u32).into())]) + } else { + // The function could raise RactorIsolationError. + jit_prepare_non_leaf_call(jit, asm); + asm.ccall(rb_ivar_get_at as *const u8, vec![recv, Opnd::UImm((ivar_index as u32).into()), Opnd::UImm(ivar_name)]) + } + }; - let out_opnd = asm.stack_push(Type::Unknown); - asm.mov(out_opnd, ivar_opnd); - } + // Push the ivar on the stack + let out_opnd = asm.stack_push(Type::Unknown); + asm.mov(out_opnd, ivar_opnd); } } @@ -3073,8 +3150,6 @@ fn gen_set_ivar( recv_opnd: YARVOpnd, ic: Option<*const iseq_inline_iv_cache_entry>, ) -> Option<CodegenStatus> { - let comptime_val_klass = comptime_receiver.class_of(); - // If the comptime receiver is frozen, writing an IV will raise an exception // and we don't want to JIT code to deal with that situation. if comptime_receiver.is_frozen() { @@ -3084,16 +3159,6 @@ fn gen_set_ivar( let stack_type = asm.ctx.get_opnd_type(StackOpnd(0)); - // Check if the comptime class uses a custom allocator - let custom_allocator = unsafe { rb_get_alloc_func(comptime_val_klass) }; - let uses_custom_allocator = match custom_allocator { - Some(alloc_fun) => { - let allocate_instance = rb_class_allocate_instance as *const u8; - alloc_fun as *const u8 != allocate_instance - } - None => false, - }; - // Check if the comptime receiver is a T_OBJECT let receiver_t_object = unsafe { RB_TYPE_P(comptime_receiver, RUBY_T_OBJECT) }; // Use a general C call at the last chain to avoid exits on megamorphic shapes @@ -3103,10 +3168,10 @@ fn gen_set_ivar( } // Get the iv index - let shape_too_complex = comptime_receiver.shape_too_complex(); - let ivar_index = if !comptime_receiver.special_const_p() && !shape_too_complex { + let shape_complex = comptime_receiver.shape_complex(); + let ivar_index = if !comptime_receiver.special_const_p() && !shape_complex { let shape_id = comptime_receiver.shape_id_of(); - let mut ivar_index: u16 = 0; + let mut ivar_index: attr_index_t = 0; if unsafe { rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) } { Some(ivar_index as usize) } else { @@ -3117,21 +3182,23 @@ fn gen_set_ivar( }; // The current shape doesn't contain this iv, we need to transition to another shape. - let mut new_shape_too_complex = false; - let new_shape = if !shape_too_complex && receiver_t_object && ivar_index.is_none() { + let mut new_shape_complex = false; + let new_shape = if !shape_complex && receiver_t_object && ivar_index.is_none() { let current_shape_id = comptime_receiver.shape_id_of(); - let next_shape_id = unsafe { rb_shape_transition_add_ivar_no_warnings(comptime_receiver, ivar_name) }; + // We don't need to check about imemo_fields here because we're definitely looking at a T_OBJECT. + let klass = unsafe { rb_obj_class(comptime_receiver) }; + let next_shape_id = unsafe { rb_shape_transition_add_ivar_no_warnings(current_shape_id, ivar_name, klass) }; // If the VM ran out of shapes, or this class generated too many leaf, - // it may be de-optimized into OBJ_TOO_COMPLEX_SHAPE (hash-table). - new_shape_too_complex = unsafe { rb_yjit_shape_too_complex_p(next_shape_id) }; - if new_shape_too_complex { + // it may be de-optimized into OBJ_COMPLEX_SHAPE (hash-table). + new_shape_complex = unsafe { rb_jit_shape_complex_p(next_shape_id) }; + if new_shape_complex { Some((next_shape_id, None, 0_usize)) } else { let current_capacity = unsafe { rb_yjit_shape_capacity(current_shape_id) }; let next_capacity = unsafe { rb_yjit_shape_capacity(next_shape_id) }; - // If the new shape has a different capacity, or is TOO_COMPLEX, we'll have to + // If the new shape has a different capacity, or is COMPLEX, we'll have to // reallocate it. let needs_extension = next_capacity != current_capacity; @@ -3149,10 +3216,9 @@ fn gen_set_ivar( None }; - // If the receiver isn't a T_OBJECT, or uses a custom allocator, - // then just write out the IV write as a function call. + // If the receiver isn't a T_OBJECT, then just write out the IV write as a function call. // too-complex shapes can't use index access, so we use rb_ivar_get for them too. - if !receiver_t_object || uses_custom_allocator || shape_too_complex || new_shape_too_complex || megamorphic { + if !receiver_t_object || shape_complex || new_shape_complex || megamorphic { // The function could raise FrozenError. // Note that this modifies REG_SP, which is why we do it first jit_prepare_non_leaf_call(jit, asm); @@ -3176,7 +3242,7 @@ fn gen_set_ivar( asm.ccall( rb_vm_setinstancevariable as *const u8, vec![ - Opnd::const_ptr(jit.iseq as *const u8), + VALUE(jit.iseq as usize).into(), Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF), ivar_name.into(), val_opnd, @@ -3369,7 +3435,7 @@ fn gen_definedivar( // Specialize base on compile time values let comptime_receiver = jit.peek_at_self(); - if comptime_receiver.special_const_p() || comptime_receiver.shape_too_complex() || asm.ctx.get_chain_depth() >= GET_IVAR_MAX_DEPTH { + if comptime_receiver.special_const_p() || comptime_receiver.shape_complex() || asm.ctx.get_chain_depth() >= GET_IVAR_MAX_DEPTH { // Fall back to calling rb_ivar_defined // Save the PC and SP because the callee may allocate @@ -3395,7 +3461,7 @@ fn gen_definedivar( let shape_id = comptime_receiver.shape_id_of(); let ivar_exists = unsafe { - let mut ivar_index: u16 = 0; + let mut ivar_index: attr_index_t = 0; rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) }; @@ -3694,7 +3760,6 @@ fn gen_equality_specialized( jit_guard_known_klass( jit, asm, - unsafe { rb_cString }, a_opnd, a_opnd.into(), comptime_a, @@ -3720,7 +3785,6 @@ fn gen_equality_specialized( jit_guard_known_klass( jit, asm, - unsafe { rb_cString }, b_opnd, b_opnd.into(), comptime_b, @@ -3817,7 +3881,6 @@ fn gen_opt_aref( jit_guard_known_klass( jit, asm, - unsafe { rb_cArray }, recv_opnd, recv_opnd.into(), comptime_recv, @@ -3857,7 +3920,6 @@ fn gen_opt_aref( jit_guard_known_klass( jit, asm, - unsafe { rb_cHash }, recv_opnd, recv_opnd.into(), comptime_recv, @@ -3888,40 +3950,6 @@ fn gen_opt_aref( } } -fn gen_opt_aset_with( - jit: &mut JITState, - asm: &mut Assembler, -) -> Option<CodegenStatus> { - // We might allocate or raise - jit_prepare_non_leaf_call(jit, asm); - - let key_opnd = Opnd::Value(jit.get_arg(0)); - let recv_opnd = asm.stack_opnd(1); - let value_opnd = asm.stack_opnd(0); - - extern "C" { - fn rb_vm_opt_aset_with(recv: VALUE, key: VALUE, value: VALUE) -> VALUE; - } - - let val_opnd = asm.ccall( - rb_vm_opt_aset_with as *const u8, - vec![ - recv_opnd, - key_opnd, - value_opnd, - ], - ); - asm.stack_pop(2); // Keep it on stack during GC - - asm.cmp(val_opnd, Qundef.into()); - asm.je(Target::side_exit(Counter::opt_aset_with_qundef)); - - let top = asm.stack_push(Type::Unknown); - asm.mov(top, val_opnd); - - return Some(KeepCompiling); -} - fn gen_opt_aset( jit: &mut JITState, asm: &mut Assembler, @@ -3944,7 +3972,6 @@ fn gen_opt_aset( jit_guard_known_klass( jit, asm, - unsafe { rb_cArray }, recv, recv.into(), comptime_recv, @@ -3956,7 +3983,6 @@ fn gen_opt_aset( jit_guard_known_klass( jit, asm, - unsafe { rb_cInteger }, key, key.into(), comptime_key, @@ -3989,7 +4015,6 @@ fn gen_opt_aset( jit_guard_known_klass( jit, asm, - unsafe { rb_cHash }, recv, recv.into(), comptime_recv, @@ -4017,38 +4042,6 @@ fn gen_opt_aset( } } -fn gen_opt_aref_with( - jit: &mut JITState, - asm: &mut Assembler, -) -> Option<CodegenStatus>{ - // We might allocate or raise - jit_prepare_non_leaf_call(jit, asm); - - let key_opnd = Opnd::Value(jit.get_arg(0)); - let recv_opnd = asm.stack_opnd(0); - - extern "C" { - fn rb_vm_opt_aref_with(recv: VALUE, key: VALUE) -> VALUE; - } - - let val_opnd = asm.ccall( - rb_vm_opt_aref_with as *const u8, - vec![ - recv_opnd, - key_opnd - ], - ); - asm.stack_pop(1); // Keep it on stack during GC - - asm.cmp(val_opnd, Qundef.into()); - asm.je(Target::side_exit(Counter::opt_aref_with_qundef)); - - let top = asm.stack_push(Type::Unknown); - asm.mov(top, val_opnd); - - return Some(KeepCompiling); -} - fn gen_opt_and( jit: &mut JITState, asm: &mut Assembler, @@ -4658,21 +4651,8 @@ fn gen_opt_case_dispatch( // Check that all cases are fixnums to avoid having to register BOP assumptions on // all the types that case hashes support. This spends compile time to save memory. - fn case_hash_all_fixnum_p(hash: VALUE) -> bool { - let mut all_fixnum = true; - unsafe { - unsafe extern "C" fn per_case(key: st_data_t, _value: st_data_t, data: st_data_t) -> c_int { - (if VALUE(key as usize).fixnum_p() { - ST_CONTINUE - } else { - (data as *mut bool).write(false); - ST_STOP - }) as c_int - } - rb_hash_stlike_foreach(hash, Some(per_case), (&mut all_fixnum) as *mut _ as st_data_t); - } - - all_fixnum + fn case_hash_all_fixnum_p(cdhash: VALUE) -> bool { + unsafe { rb_yjit_cdhash_all_fixnum_p(cdhash) } } // If megamorphic, fallback to compiling branch instructions after opt_case_dispatch @@ -4699,12 +4679,11 @@ fn gen_opt_case_dispatch( // Get the offset for the compile-time key let mut offset = 0; - unsafe { rb_hash_stlike_lookup(case_hash, comptime_key.0 as _, &mut offset) }; - let jump_offset = if offset == 0 { + let jump_offset = if unsafe { rb_yjit_cdhash_lookup(case_hash, comptime_key.0 as _, &mut offset) } == 0 { // NOTE: If we hit the else branch with various values, it could negatively impact the performance. else_offset } else { - (offset as u32) >> 1 // FIX2LONG + offset as u32 }; // Jump to the offset of case or else @@ -4725,7 +4704,7 @@ fn gen_branchif( let jump_offset = jit.get_arg(0).as_i32(); // Check for interrupts, but only on backward branches that may create loops - if jump_offset < 0 { + if jump_offset < 0 && jit.get_opcode() != YARVINSN_branchif_without_ints as usize { gen_check_ints(asm, Counter::branchif_interrupted); } @@ -4777,7 +4756,7 @@ fn gen_branchunless( let jump_offset = jit.get_arg(0).as_i32(); // Check for interrupts, but only on backward branches that may create loops - if jump_offset < 0 { + if jump_offset < 0 && jit.get_opcode() != YARVINSN_branchunless_without_ints as usize { gen_check_ints(asm, Counter::branchunless_interrupted); } @@ -4830,7 +4809,7 @@ fn gen_branchnil( let jump_offset = jit.get_arg(0).as_i32(); // Check for interrupts, but only on backward branches that may create loops - if jump_offset < 0 { + if jump_offset < 0 && jit.get_opcode() != YARVINSN_branchnil_without_ints as usize { gen_check_ints(asm, Counter::branchnil_interrupted); } @@ -4941,7 +4920,6 @@ fn gen_opt_new( perf_call!("opt_new: ", jit_guard_known_klass( jit, asm, - comptime_recv_klass, recv, recv.into(), comptime_recv, @@ -4986,7 +4964,7 @@ fn gen_jump( let jump_offset = jit.get_arg(0).as_i32(); // Check for interrupts, but only on backward branches that may create loops - if jump_offset < 0 { + if jump_offset < 0 && jit.get_opcode() != YARVINSN_jump_without_ints as usize { gen_check_ints(asm, Counter::jump_interrupted); } @@ -5012,13 +4990,13 @@ fn gen_jump( fn jit_guard_known_klass( jit: &mut JITState, asm: &mut Assembler, - known_klass: VALUE, obj_opnd: Opnd, insn_opnd: YARVOpnd, sample_instance: VALUE, max_chain_depth: u8, counter: Counter, ) { + let known_klass = sample_instance.class_of(); let val_type = asm.ctx.get_opnd_type(insn_opnd); if val_type.known_class() == Some(known_klass) { @@ -5124,7 +5102,7 @@ fn jit_guard_known_klass( assert_eq!(sample_instance.class_of(), rb_cString, "context says class is exactly ::String") }; } else { - assert!(!val_type.is_imm()); + assert!(!val_type.is_imm(), "{insn_opnd:?} should be a heap object, but was {val_type:?} for {sample_instance:?}"); // Check that the receiver is a heap object // Note: if we get here, the class doesn't have immediate instances. @@ -5768,7 +5746,6 @@ fn jit_rb_float_plus( jit_guard_known_klass( jit, asm, - comptime_obj.class_of(), obj, obj.into(), comptime_obj, @@ -5810,7 +5787,6 @@ fn jit_rb_float_minus( jit_guard_known_klass( jit, asm, - comptime_obj.class_of(), obj, obj.into(), comptime_obj, @@ -5852,7 +5828,6 @@ fn jit_rb_float_mul( jit_guard_known_klass( jit, asm, - comptime_obj.class_of(), obj, obj.into(), comptime_obj, @@ -5894,7 +5869,6 @@ fn jit_rb_float_div( jit_guard_known_klass( jit, asm, - comptime_obj.class_of(), obj, obj.into(), comptime_obj, @@ -6158,7 +6132,6 @@ fn jit_rb_str_getbyte( jit_guard_known_klass( jit, asm, - comptime_idx.class_of(), idx, idx.into(), comptime_idx, @@ -6275,7 +6248,7 @@ fn jit_rb_str_dup( jit_prepare_call_with_gc(jit, asm); - let recv_opnd = asm.stack_pop(1); + let recv_opnd = asm.stack_opnd(0); let recv_opnd = asm.load(recv_opnd); let shape_id_offset = unsafe { rb_shape_id_offset() }; @@ -6284,8 +6257,10 @@ fn jit_rb_str_dup( asm.jnz(Target::side_exit(Counter::send_str_dup_exivar)); // Call rb_str_dup - let stack_ret = asm.stack_push(Type::CString); let ret_opnd = asm.ccall(rb_str_dup as *const u8, vec![recv_opnd]); + + asm.stack_pop(1); + let stack_ret = asm.stack_push(Type::CString); asm.mov(stack_ret, ret_opnd); true @@ -6341,7 +6316,7 @@ fn jit_rb_str_concat_codepoint( guard_object_is_fixnum(jit, asm, codepoint, StackOpnd(0)); - asm.ccall(rb_yjit_str_concat_codepoint as *const u8, vec![recv, codepoint]); + asm.ccall(rb_jit_str_concat_codepoint as *const u8, vec![recv, codepoint]); // The receiver is the return value, so we only need to pop the codepoint argument off the stack. // We can reuse the receiver slot in the stack as the return value. @@ -6647,6 +6622,7 @@ fn jit_rb_f_block_given_p( true } +/// Codegen for `block_given?` and `defined?(yield)` fn gen_block_given( jit: &mut JITState, asm: &mut Assembler, @@ -6746,7 +6722,7 @@ fn jit_thread_s_current( asm.stack_pop(1); // ec->thread_ptr - let ec_thread_opnd = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR)); + let ec_thread_opnd = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR as i32)); // thread->self let thread_self = Opnd::mem(64, ec_thread_opnd, RUBY_OFFSET_THREAD_SELF); @@ -7211,7 +7187,7 @@ fn gen_send_cfunc( asm_comment!(asm, "set ec->cfp"); let new_cfp = asm.lea(Opnd::mem(64, CFP, -(RUBY_SIZEOF_CONTROL_FRAME as i32))); - asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), new_cfp); if !kw_arg.is_null() { // Build a hash from all kwargs passed @@ -7307,7 +7283,7 @@ fn gen_send_cfunc( // Pop the stack frame (ec->cfp++) // Instead of recalculating, we can reuse the previous CFP, which is stored in a callee-saved // register - let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP); + let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32); asm.store(ec_cfp_opnd, CFP); // cfunc calls may corrupt types @@ -7473,7 +7449,7 @@ fn gen_send_bmethod( ) -> Option<CodegenStatus> { let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; - let proc = unsafe { rb_yjit_get_proc_ptr(procv) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; let proc_block = unsafe { &(*proc).block }; if proc_block.type_ != block_type_iseq { @@ -7483,11 +7459,12 @@ fn gen_send_bmethod( let capture = unsafe { proc_block.as_.captured.as_ref() }; let iseq = unsafe { *capture.code.iseq.as_ref() }; - // Optimize for single ractor mode and avoid runtime check for - // "defined with an un-shareable Proc in a different Ractor" - if !assume_single_ractor_mode(jit, asm) { - gen_counter_incr(jit, asm, Counter::send_bmethod_ractor); - return None; + if !procv.shareable_p() { + let ractor_serial = unsafe { rb_yjit_cme_ractor_serial(cme) }; + asm_comment!(asm, "guard current ractor == {}", ractor_serial); + let current_ractor_serial = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_RACTOR_ID as i32)); + asm.cmp(current_ractor_serial, ractor_serial.into()); + asm.jne(Target::side_exit(Counter::send_bmethod_ractor)); } // Passing a block to a block needs logic different from passing @@ -7553,6 +7530,12 @@ fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<Opnd>, block: Opti let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32(); let local_idx = ep_offset_to_local_idx(iseq, ep_offset); + // Only inline getlocal on a parameter. DCE in the IESQ builder can + // make a two-instruction ISEQ that does not return a parameter. + if local_idx >= unsafe { get_iseq_body_param_size(iseq) } { + return None; + } + if unsafe { rb_simple_iseq_p(iseq) } { return Some(IseqReturn::LocalVariable(local_idx)); } else if unsafe { rb_iseq_only_kwparam_p(iseq) } { @@ -7709,7 +7692,7 @@ fn gen_send_iseq( gen_counter_incr(jit, asm, Counter::send_iseq_splat_not_array); return None; } else { - unsafe { rb_yjit_array_len(array) as u32} + unsafe { rb_jit_array_len(array) as u32} }; // Arity check accounting for size of the splat. When callee has rest parameters, we insert @@ -7800,7 +7783,7 @@ fn gen_send_iseq( gen_counter_incr(jit, asm, Counter::num_send_iseq); // Shortcut for special `Primitive.attr! :leaf` builtins - let builtin_attrs = unsafe { rb_yjit_iseq_builtin_attrs(iseq) }; + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; let builtin_func_raw = unsafe { rb_yjit_builtin_function(iseq) }; let builtin_func = if builtin_func_raw.is_null() { None } else { Some(builtin_func_raw) }; let opt_send_call = flags & VM_CALL_OPT_SEND != 0; // .send call is not currently supported for builtins @@ -7977,6 +7960,11 @@ fn gen_send_iseq( gen_counter_incr(jit, asm, Counter::send_iseq_clobbering_block_arg); return None; } + if iseq_has_rest || has_kwrest { + // The proc would be stored above the current stack top, where GC can't see it + gen_counter_incr(jit, asm, Counter::send_iseq_block_arg_gc_unsafe); + return None; + } let proc = asm.stack_pop(1); // Pop first, as argc doesn't account for the block arg let callee_specval = asm.ctx.sp_opnd(callee_specval); asm.store(callee_specval, proc); @@ -8389,7 +8377,7 @@ fn gen_send_iseq( // We also do this after spill_regs() to avoid doubly spilling the same thing on asm.ccall(). if get_option!(gen_stats) { // Protect caller-saved registers in case they're used for arguments - asm.cpush_all(); + let mapping = asm.cpush_all(); // Assemble the ISEQ name string let name_str = get_iseq_name(iseq); @@ -8399,7 +8387,7 @@ fn gen_send_iseq( // Increment the counter for this cfunc asm.ccall(incr_iseq_counter as *const u8, vec![iseq_idx.into()]); - asm.cpop_all(); + asm.cpop_all(mapping); } // The callee might change locals through Kernel#binding and other means. @@ -8434,7 +8422,7 @@ fn gen_send_iseq( asm_comment!(asm, "switch to new CFP"); let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); asm.mov(CFP, new_cfp); - asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); // Directly jump to the entry point of the callee gen_direct_jump( @@ -8985,9 +8973,9 @@ fn gen_struct_aref( handle_opt_send_shift_stack(asm, argc); } - // All structs from the same Struct class should have the same + // All structs from the same Struct class and shape_id should have the same // length. So if our comptime_recv is embedded all runtime - // structs of the same class should be as well, and the same is + // structs of the same class and shape_id should be as well, and the same is // true of the converse. let embedded = unsafe { FL_TEST_RAW(comptime_recv, VALUE(RSTRUCT_EMBED_LEN_MASK)) }; @@ -9022,6 +9010,12 @@ fn gen_struct_aset( return None; } + // If the comptime receiver is frozen, writing a struct member will raise an exception + // and we don't want to JIT code to deal with that situation. + if comptime_recv.is_frozen() { + return None; + } + if c_method_tracing_currently_enabled(jit) { // Struct accesses need fire c_call and c_return events, which we can't support // See :attr-tracing: @@ -9042,6 +9036,17 @@ fn gen_struct_aset( assert!(unsafe { RB_TYPE_P(comptime_recv, RUBY_T_STRUCT) }); assert!((off as i64) < unsafe { RSTRUCT_LEN(comptime_recv) }); + // Even if the comptime recv was not frozen, future recv may be. So we need to emit a guard + // that the recv is not frozen. + // We know all structs are heap objects, so we can check the flag directly. + let recv = asm.stack_opnd(1); + let recv = asm.load(recv); + let flags = asm.load(Opnd::mem(VALUE_BITS, recv, RUBY_OFFSET_RBASIC_FLAGS)); + asm.test(flags, (RUBY_FL_FREEZE as u64).into()); + asm.jnz(Target::side_exit(Counter::opt_aset_frozen)); + + // Not frozen, so we can proceed. + asm_comment!(asm, "struct aset"); let val = asm.stack_pop(1); @@ -9153,7 +9158,6 @@ fn gen_send_general( let recv_opnd: YARVOpnd = recv.into(); // Log the name of the method we're calling to - #[cfg(feature = "disasm")] asm_comment!(asm, "call to {}", get_method_name(Some(comptime_recv_klass), mid)); // Gather some statistics about sends @@ -9173,7 +9177,6 @@ fn gen_send_general( perf_call!("gen_send_general: ", jit_guard_known_klass( jit, asm, - comptime_recv_klass, recv, recv_opnd, comptime_recv, @@ -9441,13 +9444,6 @@ fn gen_send_general( return None; } - // Optimize for single ractor mode and avoid runtime check for - // "defined with an un-shareable Proc in a different Ractor" - if !assume_single_ractor_mode(jit, asm) { - gen_counter_incr(jit, asm, Counter::send_call_multi_ractor); - return None; - } - // If this is a .send call we need to adjust the stack if flags & VM_CALL_OPT_SEND != 0 { handle_opt_send_shift_stack(asm, argc); @@ -9652,7 +9648,24 @@ fn gen_sendforward( jit: &mut JITState, asm: &mut Assembler, ) -> Option<CodegenStatus> { - return gen_send(jit, asm); + // Generate specialized code if possible + let cd = jit.get_arg(0).as_ptr(); + let block = jit.get_arg(1).as_optional_ptr().map(|iseq| BlockHandler::BlockISeq(iseq)); + if let Some(status) = perf_call! { gen_send_general(jit, asm, cd, block) } { + return Some(status); + } + + // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of sendforward + let blockiseq = jit.get_arg(1).as_iseq(); + gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { + extern "C" { + fn rb_vm_sendforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm.ccall( + rb_vm_sendforward as *const u8, + vec![EC, CFP, (cd as usize).into(), VALUE(blockiseq as usize).into()], + ) + }) } fn gen_invokeblock( @@ -9726,7 +9739,7 @@ fn gen_invokeblock_specialized( // If the current ISEQ is annotated to be inlined but it's not being inlined here, // generate a dynamic dispatch to avoid making this yield megamorphic. - if unsafe { rb_yjit_iseq_builtin_attrs(jit.iseq) } & BUILTIN_ATTR_INLINE_BLOCK != 0 && !asm.ctx.inline() { + if unsafe { rb_jit_iseq_builtin_attrs(jit.iseq) } & BUILTIN_ATTR_INLINE_BLOCK != 0 && !asm.ctx.inline() { gen_counter_incr(jit, asm, Counter::invokeblock_iseq_not_inlined); return None; } @@ -9817,7 +9830,7 @@ fn gen_invokesuper( return Some(status); } - // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of send + // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of invokesuper let blockiseq = jit.get_arg(1).as_iseq(); gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { extern "C" { @@ -9834,7 +9847,23 @@ fn gen_invokesuperforward( jit: &mut JITState, asm: &mut Assembler, ) -> Option<CodegenStatus> { - return gen_invokesuper(jit, asm); + // Generate specialized code if possible + let cd = jit.get_arg(0).as_ptr(); + if let Some(status) = gen_invokesuper_specialized(jit, asm, cd) { + return Some(status); + } + + // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of invokesuperforward + let blockiseq = jit.get_arg(1).as_iseq(); + gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { + extern "C" { + fn rb_vm_invokesuperforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm.ccall( + rb_vm_invokesuperforward as *const u8, + vec![EC, CFP, (cd as usize).into(), VALUE(blockiseq as usize).into()], + ) + }) } fn gen_invokesuper_specialized( @@ -9866,6 +9895,15 @@ fn gen_invokesuper_specialized( return None; } + let ci = unsafe { get_call_data_ci(cd) }; + let ci_flags = unsafe { vm_ci_flag(ci) }; + + // Bail on ZSUPER inside a block method. They always raise. + if ci_flags & VM_CALL_ZSUPER != 0 && VM_METHOD_TYPE_BMETHOD == unsafe { get_cme_def_type(me) } { + gen_counter_incr(jit, asm, Counter::invokesuper_bmethod_zsuper); + return None; + } + // FIXME: We should track and invalidate this block when this cme is invalidated let current_defined_class = unsafe { (*me).defined_class }; let mid = unsafe { get_def_original_id((*me).def) }; @@ -9881,14 +9919,8 @@ fn gen_invokesuper_specialized( let comptime_superclass = unsafe { rb_class_get_superclass(RCLASS_ORIGIN(current_defined_class)) }; - let ci = unsafe { get_call_data_ci(cd) }; - let argc: i32 = unsafe { vm_ci_argc(ci) }.try_into().unwrap(); - - let ci_flags = unsafe { vm_ci_flag(ci) }; - // Don't JIT calls that aren't simple // Note, not using VM_CALL_ARGS_SIMPLE because sometimes we pass a block. - if ci_flags & VM_CALL_KWARG != 0 { gen_counter_incr(jit, asm, Counter::invokesuper_kwarg); return None; @@ -9907,6 +9939,7 @@ fn gen_invokesuper_specialized( // cheaper calculations first, but since we specialize on the method entry // and so only have to do this once at compile time this is fine to always // check and side exit. + let argc: i32 = unsafe { vm_ci_argc(ci) }.try_into().unwrap(); let comptime_recv = jit.peek_at_stack(&asm.ctx, argc as isize); if unsafe { rb_obj_is_kind_of(comptime_recv, current_defined_class) } == VALUE(0) { gen_counter_incr(jit, asm, Counter::invokesuper_defined_class_mismatch); @@ -9934,16 +9967,8 @@ fn gen_invokesuper_specialized( return None; } - asm_comment!(asm, "guard known me"); - let lep_opnd = gen_get_lep(jit, asm); - let ep_me_opnd = Opnd::mem( - 64, - lep_opnd, - SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF, - ); - - let me_as_value = VALUE(me as usize); - asm.cmp(ep_me_opnd, me_as_value.into()); + let cme_opnd = gen_get_running_cme_or_sentinal(jit, asm); + asm.cmp(cme_opnd, VALUE::from(me).into()); jit_chain_guard( JCC_JNE, jit, @@ -9988,7 +10013,7 @@ fn gen_leave( asm_comment!(asm, "pop stack frame"); let incr_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); asm.mov(CFP, incr_cfp); - asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP); + asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); // Load the return value let retval_opnd = asm.stack_pop(1); @@ -10087,7 +10112,6 @@ fn gen_objtostring( jit_guard_known_klass( jit, asm, - comptime_recv.class_of(), recv, recv.into(), comptime_recv, @@ -10101,7 +10125,6 @@ fn gen_objtostring( jit_guard_known_klass( jit, asm, - comptime_recv.class_of(), recv, recv.into(), comptime_recv, @@ -10157,45 +10180,18 @@ fn gen_toregexp( let opt = jit.get_arg(0).as_i64(); let cnt = jit.get_arg(1).as_usize(); - // Save the PC and SP because this allocates an object and could - // raise an exception. + // Allocates objects and could raise an exception. jit_prepare_non_leaf_call(jit, asm); let values_ptr = asm.lea(asm.ctx.sp_opnd(-(cnt as i32))); - let ary = asm.ccall( - rb_ary_tmp_new_from_values as *const u8, - vec![ - Opnd::Imm(0), - cnt.into(), - values_ptr, - ] + let regexp = asm.ccall( + rb_reg_new_from_values as _, + vec![cnt.into(), values_ptr, opt.into()], ); asm.stack_pop(cnt); // Let ccall spill them - - // Save the array so we can clear it later - asm.cpush(ary); - asm.cpush(ary); // Alignment - - let val = asm.ccall( - rb_reg_new_ary as *const u8, - vec![ - ary, - Opnd::Imm(opt), - ] - ); - - // The actual regex is in RAX now. Pop the temp array from - // rb_ary_tmp_new_from_values into C arg regs so we can clear it - let ary = asm.cpop(); // Alignment - asm.cpop_into(ary); - - // The value we want to push on the stack is in RAX right now let stack_ret = asm.stack_push(Type::UnknownHeap); - asm.mov(stack_ret, val); - - // Clear the temp array. - asm.ccall(rb_ary_clear as *const u8, vec![ary]); + asm.mov(stack_ret, regexp); Some(KeepCompiling) } @@ -10285,7 +10281,7 @@ fn gen_getclassvariable( let val_opnd = asm.ccall( rb_vm_getclassvariable as *const u8, vec![ - Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), + VALUE(jit.iseq as usize).into(), CFP, Opnd::UImm(jit.get_arg(0).as_u64()), Opnd::UImm(jit.get_arg(1).as_u64()), @@ -10309,7 +10305,7 @@ fn gen_setclassvariable( asm.ccall( rb_vm_setclassvariable as *const u8, vec![ - Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), + VALUE(jit.iseq as usize).into(), CFP, Opnd::UImm(jit.get_arg(0).as_u64()), val, @@ -10777,8 +10773,8 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> { YARVINSN_concattoarray => Some(gen_concattoarray), YARVINSN_pushtoarray => Some(gen_pushtoarray), YARVINSN_newrange => Some(gen_newrange), - YARVINSN_putstring => Some(gen_putstring), - YARVINSN_putchilledstring => Some(gen_putchilledstring), + YARVINSN_dupstring => Some(gen_dupstring), + YARVINSN_dupchilledstring => Some(gen_dupchilledstring), YARVINSN_expandarray => Some(gen_expandarray), YARVINSN_defined => Some(gen_defined), YARVINSN_definedivar => Some(gen_definedivar), @@ -10792,8 +10788,6 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> { YARVINSN_opt_neq => Some(gen_opt_neq), YARVINSN_opt_aref => Some(gen_opt_aref), YARVINSN_opt_aset => Some(gen_opt_aset), - YARVINSN_opt_aref_with => Some(gen_opt_aref_with), - YARVINSN_opt_aset_with => Some(gen_opt_aset_with), YARVINSN_opt_mult => Some(gen_opt_mult), YARVINSN_opt_div => Some(gen_opt_div), YARVINSN_opt_ltlt => Some(gen_opt_ltlt), @@ -10815,6 +10809,10 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> { YARVINSN_branchnil => Some(gen_branchnil), YARVINSN_throw => Some(gen_throw), YARVINSN_jump => Some(gen_jump), + YARVINSN_branchif_without_ints => Some(gen_branchif), + YARVINSN_branchunless_without_ints => Some(gen_branchunless), + YARVINSN_branchnil_without_ints => Some(gen_branchnil), + YARVINSN_jump_without_ints => Some(gen_jump), YARVINSN_opt_new => Some(gen_opt_new), YARVINSN_getblockparamproxy => Some(gen_getblockparamproxy), @@ -11025,7 +11023,7 @@ impl CodegenGlobals { #[cfg(not(test))] let (mut cb, mut ocb) = { - let virt_block: *mut u8 = unsafe { rb_yjit_reserve_addr_space(exec_mem_size as u32) }; + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(exec_mem_size as u32) }; // Memory protection syscalls need page-aligned addresses, so check it here. Assuming // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the @@ -11034,7 +11032,7 @@ impl CodegenGlobals { // // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. - let page_size = unsafe { rb_yjit_get_page_size() }; + let page_size = unsafe { rb_jit_get_page_size() }; assert_eq!( virt_block as usize % page_size.as_usize(), 0, "Start of virtual address block should be page-aligned", diff --git a/yjit/src/core.rs b/yjit/src/core.rs index d42726bcc7..d08cd1fb26 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -34,7 +34,7 @@ use crate::invariants::*; pub const MAX_CTX_TEMPS: usize = 8; // Maximum number of local variable types or registers we keep track of -const MAX_CTX_LOCALS: usize = 8; +pub const MAX_CTX_LOCALS: usize = 8; /// An index into `ISEQ_BODY(iseq)->iseq_encoded`. Points /// to a YARV instruction or an instruction operand. @@ -1099,7 +1099,7 @@ impl Context { MapToLocal(local_idx) => { bits.push_op(CtxOp::MapTempLocal); bits.push_u3(stack_idx as u8); - bits.push_u3(local_idx as u8); + bits.push_u3(local_idx); } MapToSelf => { @@ -1769,6 +1769,35 @@ impl IseqPayload { // Turn it into an iterator that owns the blocks and return version_map.into_iter().flatten() } + + /// Get or create all blocks for a particular place in an iseq. + fn get_or_create_version_list(&mut self, insn_idx: usize) -> &mut VersionList { + if insn_idx >= self.version_map.len() { + self.version_map.resize(insn_idx + 1, VersionList::default()); + } + + return self.version_map.get_mut(insn_idx).unwrap(); + } + + // We cannot deallocate blocks immediately after invalidation since patching + // the code for setting up return addresses does not affect outstanding return + // addresses that are on stack and will use invalidated branch pointers when + // hit. Example: + // def foo(n) + // if n == 2 + // # 1.times.each to create a cfunc frame to preserve the JIT frame + // # which will return to a stub housed in an invalidated block + // return 1.times.each { Object.define_method(:foo) {} } + // end + // + // foo(n + 1) # The block for this call houses the return branch stub + // end + // p foo(1) + pub fn delayed_deallocation(&mut self, blockref: BlockRef) { + block_assumptions_free(blockref); + unsafe { blockref.as_ref() }.set_iseq_null(); + self.dead_blocks.push(blockref); + } } /// Get the payload for an iseq. For safety it's up to the caller to ensure the returned `&mut` @@ -1814,22 +1843,24 @@ pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { pub fn for_each_iseq<F: FnMut(IseqPtr)>(mut callback: F) { unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) { // SAFETY: points to the local below - let callback: &mut &mut dyn FnMut(IseqPtr) -> bool = unsafe { std::mem::transmute(&mut *data) }; - callback(iseq); + let callback: *mut *mut dyn FnMut(IseqPtr) -> bool = data.cast(); + unsafe { (**callback)(iseq) }; } - let mut data: &mut dyn FnMut(IseqPtr) = &mut callback; - unsafe { rb_yjit_for_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) }; + let mut data: *mut dyn FnMut(IseqPtr) = &mut callback; + let data: *mut *mut dyn FnMut(IseqPtr) = &mut data; + unsafe { rb_jit_for_each_iseq(Some(callback_wrapper), data.cast()) }; } /// Iterate over all on-stack ISEQs pub fn for_each_on_stack_iseq<F: FnMut(IseqPtr)>(mut callback: F) { unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) { // SAFETY: points to the local below - let callback: &mut &mut dyn FnMut(IseqPtr) -> bool = unsafe { std::mem::transmute(&mut *data) }; - callback(iseq); + let callback: *mut *mut dyn FnMut(IseqPtr) -> bool = data.cast(); + unsafe { (**callback)(iseq) }; } - let mut data: &mut dyn FnMut(IseqPtr) = &mut callback; - unsafe { rb_jit_cont_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) }; + let mut data: *mut dyn FnMut(IseqPtr) = &mut callback; + let data: *mut *mut dyn FnMut(IseqPtr) = &mut data; + unsafe { rb_jit_cont_each_iseq(Some(callback_wrapper), data.cast()) }; } /// Iterate over all on-stack ISEQ payloads @@ -2140,21 +2171,6 @@ fn get_version_list(blockid: BlockId) -> Option<&'static mut VersionList> { } } -/// Get or create all blocks for a particular place in an iseq. -fn get_or_create_version_list(blockid: BlockId) -> &'static mut VersionList { - let payload = get_or_create_iseq_payload(blockid.iseq); - let insn_idx = blockid.idx.as_usize(); - - // Expand the version map as necessary - if insn_idx >= payload.version_map.len() { - payload - .version_map - .resize(insn_idx + 1, VersionList::default()); - } - - return payload.version_map.get_mut(insn_idx).unwrap(); -} - /// Take all of the blocks for a particular place in an iseq pub fn take_version_list(blockid: BlockId) -> VersionList { let insn_idx = blockid.idx.as_usize(); @@ -2343,15 +2359,21 @@ unsafe fn add_block_version(blockref: BlockRef, cb: &CodeBlock) { // Function entry blocks must have stack size 0 debug_assert!(!(block.iseq_range.start == 0 && Context::decode(block.ctx).stack_size > 0)); - let version_list = get_or_create_version_list(block.get_blockid()); + // Use a single payload reference for both version_map and pages access + // to avoid mutable aliasing UB from multiple get_iseq_payload calls. + let iseq_payload = get_or_create_iseq_payload(block.iseq.get()); + let insn_idx = block.get_blockid().idx.as_usize(); - // If this the first block being compiled with this block id - if version_list.len() == 0 { - incr_counter!(compiled_blockid_count); - } + { + let version_list = iseq_payload.get_or_create_version_list(insn_idx); - version_list.push(blockref); - version_list.shrink_to_fit(); + if version_list.is_empty() { + incr_counter!(compiled_blockid_count); + } + + version_list.push(blockref); + version_list.shrink_to_fit(); + } // By writing the new block to the iseq, the iseq now // contains new references to Ruby objects. Run write barriers. @@ -2376,7 +2398,6 @@ unsafe fn add_block_version(blockref: BlockRef, cb: &CodeBlock) { } // Mark code pages for code GC - let iseq_payload = get_iseq_payload(block.iseq.get()).unwrap(); for page in cb.addrs_to_pages(block.start_addr, block.end_addr.get()) { iseq_payload.pages.insert(page); } @@ -2419,7 +2440,9 @@ impl<'a> JITState<'a> { // Pending branches => actual branches outgoing: MutableBranchList(Cell::new(self.pending_outgoing.into_iter().map(|pending_out| { let pending_out = Rc::try_unwrap(pending_out) - .ok().expect("all PendingBranchRefs should be unique when ready to construct a Block"); + .unwrap_or_else(|rc| panic!( + "PendingBranchRef should be unique when ready to construct a Block. \ + strong={} weak={}", Rc::strong_count(&rc), Rc::weak_count(&rc))); pending_out.into_branch(NonNull::new(blockref as *mut Block).expect("no null from Box")) }).collect())) }); @@ -2427,7 +2450,7 @@ impl<'a> JITState<'a> { // SAFETY: allocated with Box above unsafe { ptr::write(blockref, block) }; - // Block is initialized now. Note that MaybeUnint<T> has the same layout as T. + // Block is initialized now. Note that MaybeUninit<T> has the same layout as T. let blockref = NonNull::new(blockref as *mut Block).expect("no null from Box"); // Track all the assumptions the block makes as invariants @@ -2493,6 +2516,11 @@ impl Block { self.incoming.push(branch); } + /// Mark this block as dead by setting its iseq to null. + pub fn set_iseq_null(&self) { + self.iseq.replace(ptr::null()); + } + // Compute the size of the block code pub fn code_size(&self) -> usize { (self.end_addr.get().as_offset() - self.start_addr.as_offset()).try_into().unwrap() @@ -3591,6 +3619,13 @@ fn branch_stub_hit_body(branch_ptr: *const c_void, target_idx: u32, ec: EcPtr) - return CodegenGlobals::get_stub_exit_code().raw_ptr(cb); } + // Bail if this branch is housed in an invalidated (dead) block. + // This only happens in rare invalidation scenarios and we need + // to avoid linking a dead block to a live block with a branch. + if branch.block.get().as_ref().iseq.get().is_null() { + return CodegenGlobals::get_stub_exit_code().raw_ptr(cb); + } + (cfp, original_interp_sp) }; @@ -3790,7 +3825,7 @@ pub fn gen_branch_stub_hit_trampoline(ocb: &mut OutlinedCb) -> Option<CodePtr> { let mut asm = Assembler::new_without_iseq(); // For `branch_stub_hit(branch_ptr, target_idx, ec)`, - // `branch_ptr` and `target_idx` is different for each stub, + // `branch_ptr` and `target_idx` are different for each stub, // but the call and what's after is the same. This trampoline // is the unchanging part. // Since this trampoline is static, it allows code GC inside @@ -4289,7 +4324,9 @@ pub fn invalidate_block_version(blockref: &BlockRef) { // in this function before we removed it, so it's well connected. unsafe { remove_from_graph(*blockref) }; - delayed_deallocation(*blockref); + if let Some(payload) = get_iseq_payload(id_being_invalidated.iseq) { + payload.delayed_deallocation(*blockref); + } ocb.unwrap().mark_all_executable(); cb.mark_all_executable(); @@ -4297,28 +4334,6 @@ pub fn invalidate_block_version(blockref: &BlockRef) { incr_counter!(invalidation_count); } -// We cannot deallocate blocks immediately after invalidation since there -// could be stubs waiting to access branch pointers. Return stubs can do -// this since patching the code for setting up return addresses does not -// affect old return addresses that are already set up to use potentially -// invalidated branch pointers. Example: -// def foo(n) -// if n == 2 -// # 1.times.each to create a cfunc frame to preserve the JIT frame -// # which will return to a stub housed in an invalidated block -// return 1.times.each { Object.define_method(:foo) {} } -// end -// -// foo(n + 1) -// end -// p foo(1) -pub fn delayed_deallocation(blockref: BlockRef) { - block_assumptions_free(blockref); - - let payload = get_iseq_payload(unsafe { blockref.as_ref() }.iseq.get()).unwrap(); - payload.dead_blocks.push(blockref); -} - trait RefUnchecked { type Contained; unsafe fn ref_unchecked(&self) -> &Self::Contained; diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index f7a08b3b18..8b99a8154a 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -123,7 +123,6 @@ extern "C" { pub fn rb_float_new(d: f64) -> VALUE; pub fn rb_hash_empty_p(hash: VALUE) -> VALUE; - pub fn rb_yjit_str_concat_codepoint(str: VALUE, codepoint: VALUE); pub fn rb_str_setbyte(str: VALUE, index: VALUE, value: VALUE) -> VALUE; pub fn rb_vm_splat_array(flag: VALUE, ary: VALUE) -> VALUE; pub fn rb_vm_concat_array(ary1: VALUE, ary2st: VALUE) -> VALUE; @@ -198,8 +197,8 @@ pub use rb_get_cikw_keywords_idx as get_cikw_keywords_idx; pub use rb_get_call_data_ci as get_call_data_ci; pub use rb_yarv_str_eql_internal as rb_str_eql_internal; pub use rb_yarv_ary_entry_internal as rb_ary_entry_internal; -pub use rb_yjit_fix_div_fix as rb_fix_div_fix; -pub use rb_yjit_fix_mod_fix as rb_fix_mod_fix; +pub use rb_jit_fix_div_fix as rb_fix_div_fix; +pub use rb_jit_fix_mod_fix as rb_fix_mod_fix; pub use rb_FL_TEST as FL_TEST; pub use rb_FL_TEST_RAW as FL_TEST_RAW; pub use rb_RB_TYPE_P as RB_TYPE_P; @@ -362,6 +361,11 @@ impl VALUE { !self.special_const_p() } + /// Shareability between ractors. `RB_OBJ_SHAREABLE_P()`. + pub fn shareable_p(self) -> bool { + (self.builtin_flags() & RUBY_FL_SHAREABLE as usize) != 0 + } + /// Return true if the value is a Ruby Fixnum (immediate-size integer) pub fn fixnum_p(self) -> bool { let VALUE(cval) = self; @@ -440,8 +444,8 @@ impl VALUE { unsafe { rb_obj_frozen_p(self) != VALUE(0) } } - pub fn shape_too_complex(self) -> bool { - unsafe { rb_yjit_shape_obj_too_complex_p(self) } + pub fn shape_complex(self) -> bool { + unsafe { rb_yjit_shape_obj_complex_p(self) } } pub fn shape_id_of(self) -> u32 { @@ -450,7 +454,7 @@ impl VALUE { pub fn embedded_p(self) -> bool { unsafe { - FL_TEST_RAW(self, VALUE(ROBJECT_EMBED as usize)) != VALUE(0) + FL_TEST_RAW(self, VALUE(ROBJECT_HEAP as usize)) == VALUE(0) } } @@ -594,6 +598,13 @@ impl From<VALUE> for u16 { } } +/// Check whether a control frame has an escaped environment +pub unsafe fn cfp_env_has_escaped(cfp: *mut rb_control_frame_struct) -> bool { + use crate::utils::IntoUsize; + let ep = get_cfp_ep(cfp); + 0 != ep.offset(VM_ENV_DATA_INDEX_FLAGS as isize).read().0 & VM_ENV_FLAG_ESCAPED.as_usize() +} + /// Produce a Ruby string from a Rust string slice pub fn rust_str_to_ruby(str: &str) -> VALUE { unsafe { rb_utf8_str_new(str.as_ptr() as *const _, str.len() as i64) } @@ -677,7 +688,7 @@ where let line = loc.line; let mut recursive_lock_level: c_uint = 0; - unsafe { rb_yjit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) }; + unsafe { rb_jit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) }; let ret = match catch_unwind(func) { Ok(result) => result, @@ -697,7 +708,7 @@ where } }; - unsafe { rb_yjit_vm_unlock(&mut recursive_lock_level, file, line) }; + unsafe { rb_jit_vm_unlock(&mut recursive_lock_level, file, line) }; ret } @@ -768,12 +779,6 @@ mod manual_defs { pub const RUBY_OFFSET_CFP_JIT_RETURN: i32 = 48; pub const RUBY_SIZEOF_CONTROL_FRAME: usize = 56; - // Constants from rb_execution_context_t vm_core.h - pub const RUBY_OFFSET_EC_CFP: i32 = 16; - pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: i32 = 32; // rb_atomic_t (u32) - pub const RUBY_OFFSET_EC_INTERRUPT_MASK: i32 = 36; // rb_atomic_t (u32) - pub const RUBY_OFFSET_EC_THREAD_PTR: i32 = 48; - // Constants from rb_thread_t in vm_core.h pub const RUBY_OFFSET_THREAD_SELF: i32 = 16; @@ -816,6 +821,7 @@ pub(crate) mod ids { def_ids! { name: NULL content: b"" name: respond_to_missing content: b"respond_to_missing?" + name: method_missing content: b"method_missing" name: to_ary content: b"to_ary" name: to_s content: b"to_s" name: eq content: b"==" diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index c8a58f424e..272c10fde9 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -165,6 +165,7 @@ pub const NIL_REDEFINED_OP_FLAG: u32 = 512; pub const TRUE_REDEFINED_OP_FLAG: u32 = 1024; pub const FALSE_REDEFINED_OP_FLAG: u32 = 2048; pub const PROC_REDEFINED_OP_FLAG: u32 = 4096; +pub const VM_KW_SPECIFIED_BITS_MAX: u32 = 31; pub const VM_ENV_DATA_SIZE: u32 = 3; pub const VM_ENV_DATA_INDEX_ME_CREF: i32 = -2; pub const VM_ENV_DATA_INDEX_SPECVAL: i32 = -1; @@ -224,11 +225,9 @@ pub const RUBY_FL_WB_PROTECTED: ruby_fl_type = 32; pub const RUBY_FL_PROMOTED: ruby_fl_type = 32; pub const RUBY_FL_UNUSED6: ruby_fl_type = 64; pub const RUBY_FL_FINALIZE: ruby_fl_type = 128; -pub const RUBY_FL_TAINT: ruby_fl_type = 0; pub const RUBY_FL_EXIVAR: ruby_fl_type = 0; pub const RUBY_FL_SHAREABLE: ruby_fl_type = 256; -pub const RUBY_FL_UNTRUSTED: ruby_fl_type = 0; -pub const RUBY_FL_UNUSED9: ruby_fl_type = 512; +pub const RUBY_FL_WEAK_REFERENCE: ruby_fl_type = 512; pub const RUBY_FL_UNUSED10: ruby_fl_type = 1024; pub const RUBY_FL_FREEZE: ruby_fl_type = 2048; pub const RUBY_FL_USER0: ruby_fl_type = 4096; @@ -259,46 +258,12 @@ pub const RSTRING_FSTR: ruby_rstring_flags = 536870912; pub type ruby_rstring_flags = u32; pub type st_data_t = ::std::os::raw::c_ulong; pub type st_index_t = st_data_t; -#[repr(C)] -#[derive(Debug, Copy, Clone)] -pub struct st_hash_type { - pub compare: ::std::option::Option< - unsafe extern "C" fn(arg1: st_data_t, arg2: st_data_t) -> ::std::os::raw::c_int, - >, - pub hash: ::std::option::Option<unsafe extern "C" fn(arg1: st_data_t) -> st_index_t>, -} -#[repr(C)] -#[derive(Debug, Copy, Clone)] -pub struct st_table_entry { - _unused: [u8; 0], -} -#[repr(C)] -#[derive(Debug, Copy, Clone)] -pub struct st_table { - pub entry_power: ::std::os::raw::c_uchar, - pub bin_power: ::std::os::raw::c_uchar, - pub size_ind: ::std::os::raw::c_uchar, - pub rebuilds_num: ::std::os::raw::c_uint, - pub type_: *const st_hash_type, - pub num_entries: st_index_t, - pub bins: *mut st_index_t, - pub entries_start: st_index_t, - pub entries_bound: st_index_t, - pub entries: *mut st_table_entry, -} pub const ST_CONTINUE: st_retval = 0; pub const ST_STOP: st_retval = 1; pub const ST_DELETE: st_retval = 2; pub const ST_CHECK: st_retval = 3; pub const ST_REPLACE: st_retval = 4; pub type st_retval = u32; -pub type st_foreach_callback_func = ::std::option::Option< - unsafe extern "C" fn( - arg1: st_data_t, - arg2: st_data_t, - arg3: st_data_t, - ) -> ::std::os::raw::c_int, ->; pub const RARRAY_EMBED_FLAG: ruby_rarray_flags = 8192; pub const RARRAY_EMBED_LEN_MASK: ruby_rarray_flags = 4161536; pub type ruby_rarray_flags = u32; @@ -306,7 +271,7 @@ pub const RARRAY_EMBED_LEN_SHIFT: ruby_rarray_consts = 15; pub type ruby_rarray_consts = u32; pub const RMODULE_IS_REFINEMENT: ruby_rmodule_flags = 8192; pub type ruby_rmodule_flags = u32; -pub const ROBJECT_EMBED: ruby_robject_flags = 8192; +pub const ROBJECT_HEAP: ruby_robject_flags = 65536; pub type ruby_robject_flags = u32; pub type rb_block_call_func = ::std::option::Option< unsafe extern "C" fn( @@ -356,45 +321,24 @@ pub const BOP_NIL_P: ruby_basic_operators = 15; pub const BOP_SUCC: ruby_basic_operators = 16; pub const BOP_GT: ruby_basic_operators = 17; pub const BOP_GE: ruby_basic_operators = 18; -pub const BOP_NOT: ruby_basic_operators = 19; -pub const BOP_NEQ: ruby_basic_operators = 20; -pub const BOP_MATCH: ruby_basic_operators = 21; -pub const BOP_FREEZE: ruby_basic_operators = 22; -pub const BOP_UMINUS: ruby_basic_operators = 23; -pub const BOP_MAX: ruby_basic_operators = 24; -pub const BOP_MIN: ruby_basic_operators = 25; -pub const BOP_HASH: ruby_basic_operators = 26; -pub const BOP_CALL: ruby_basic_operators = 27; -pub const BOP_AND: ruby_basic_operators = 28; -pub const BOP_OR: ruby_basic_operators = 29; -pub const BOP_CMP: ruby_basic_operators = 30; -pub const BOP_DEFAULT: ruby_basic_operators = 31; -pub const BOP_PACK: ruby_basic_operators = 32; -pub const BOP_INCLUDE_P: ruby_basic_operators = 33; -pub const BOP_LAST_: ruby_basic_operators = 34; +pub const BOP_GTGT: ruby_basic_operators = 19; +pub const BOP_NOT: ruby_basic_operators = 20; +pub const BOP_NEQ: ruby_basic_operators = 21; +pub const BOP_MATCH: ruby_basic_operators = 22; +pub const BOP_FREEZE: ruby_basic_operators = 23; +pub const BOP_UMINUS: ruby_basic_operators = 24; +pub const BOP_MAX: ruby_basic_operators = 25; +pub const BOP_MIN: ruby_basic_operators = 26; +pub const BOP_HASH: ruby_basic_operators = 27; +pub const BOP_CALL: ruby_basic_operators = 28; +pub const BOP_AND: ruby_basic_operators = 29; +pub const BOP_OR: ruby_basic_operators = 30; +pub const BOP_CMP: ruby_basic_operators = 31; +pub const BOP_DEFAULT: ruby_basic_operators = 32; +pub const BOP_PACK: ruby_basic_operators = 33; +pub const BOP_INCLUDE_P: ruby_basic_operators = 34; +pub const BOP_LAST_: ruby_basic_operators = 35; pub type ruby_basic_operators = u32; -#[repr(C)] -pub struct rb_namespace_struct { - pub ns_object: VALUE, - pub ns_id: ::std::os::raw::c_long, - pub top_self: VALUE, - pub load_path: VALUE, - pub load_path_snapshot: VALUE, - pub load_path_check_cache: VALUE, - pub expanded_load_path: VALUE, - pub loaded_features: VALUE, - pub loaded_features_snapshot: VALUE, - pub loaded_features_realpaths: VALUE, - pub loaded_features_realpath_map: VALUE, - pub loaded_features_index: *mut st_table, - pub loading_table: *mut st_table, - pub ruby_dln_libmap: VALUE, - pub gvar_tbl: VALUE, - pub is_builtin: bool, - pub is_user: bool, - pub is_optional: bool, -} -pub type rb_namespace_t = rb_namespace_struct; pub type rb_serial_t = ::std::os::raw::c_ulonglong; pub const imemo_env: imemo_type = 0; pub const imemo_cref: imemo_type = 1; @@ -405,10 +349,13 @@ pub const imemo_memo: imemo_type = 5; pub const imemo_ment: imemo_type = 6; pub const imemo_iseq: imemo_type = 7; pub const imemo_tmpbuf: imemo_type = 8; +pub const imemo_cvar_entry: imemo_type = 9; pub const imemo_callinfo: imemo_type = 10; pub const imemo_callcache: imemo_type = 11; pub const imemo_constcache: imemo_type = 12; pub const imemo_fields: imemo_type = 13; +pub const imemo_subclasses: imemo_type = 14; +pub const imemo_cdhash: imemo_type = 15; pub type imemo_type = u32; #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -424,6 +371,7 @@ pub struct vm_ifunc { pub data: *const ::std::os::raw::c_void, pub argc: vm_ifunc_argc, } +pub type rb_atomic_t = ::std::os::raw::c_uint; pub const METHOD_VISI_UNDEF: rb_method_visibility_t = 0; pub const METHOD_VISI_PUBLIC: rb_method_visibility_t = 1; pub const METHOD_VISI_PRIVATE: rb_method_visibility_t = 2; @@ -502,8 +450,6 @@ pub type ruby_vm_throw_flags = u32; pub struct iseq_inline_constant_cache_entry { pub flags: VALUE, pub value: VALUE, - pub _unused1: VALUE, - pub _unused2: VALUE, pub ic_cref: *const rb_cref_t, } #[repr(C)] @@ -537,10 +483,11 @@ pub const BUILTIN_ATTR_LEAF: rb_builtin_attr = 1; pub const BUILTIN_ATTR_SINGLE_NOARG_LEAF: rb_builtin_attr = 2; pub const BUILTIN_ATTR_INLINE_BLOCK: rb_builtin_attr = 4; pub const BUILTIN_ATTR_C_TRACE: rb_builtin_attr = 8; +pub const BUILTIN_ATTR_WITHOUT_INTERRUPTS: rb_builtin_attr = 16; pub type rb_builtin_attr = u32; #[repr(C)] #[derive(Debug, Copy, Clone)] -pub struct rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword { +pub struct rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword { pub num: ::std::os::raw::c_int, pub required_num: ::std::os::raw::c_int, pub bits_start: ::std::os::raw::c_int, @@ -582,7 +529,6 @@ pub type rb_control_frame_t = rb_control_frame_struct; #[repr(C)] pub struct rb_proc_t { pub block: rb_block, - pub ns: *const rb_namespace_t, pub _bitfield_align_1: [u8; 0], pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>, pub __bindgen_padding_0: [u8; 7usize], @@ -678,19 +624,19 @@ pub const VM_FRAME_FLAG_LAMBDA: vm_frame_env_flags = 256; pub const VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM: vm_frame_env_flags = 512; pub const VM_FRAME_FLAG_CFRAME_KW: vm_frame_env_flags = 1024; pub const VM_FRAME_FLAG_PASSED: vm_frame_env_flags = 2048; -pub const VM_FRAME_FLAG_NS_SWITCH: vm_frame_env_flags = 4096; -pub const VM_FRAME_FLAG_LOAD_ISEQ: vm_frame_env_flags = 8192; +pub const VM_FRAME_FLAG_BOX_REQUIRE: vm_frame_env_flags = 4096; pub const VM_ENV_FLAG_LOCAL: vm_frame_env_flags = 2; pub const VM_ENV_FLAG_ESCAPED: vm_frame_env_flags = 4; pub const VM_ENV_FLAG_WB_REQUIRED: vm_frame_env_flags = 8; pub const VM_ENV_FLAG_ISOLATED: vm_frame_env_flags = 16; pub type vm_frame_env_flags = u32; -pub type attr_index_t = u16; +pub type attr_index_t = u8; pub type shape_id_t = u32; -pub const SHAPE_ID_HAS_IVAR_MASK: _bindgen_ty_37 = 134742014; -pub type _bindgen_ty_37 = u32; +pub const SHAPE_ID_HAS_IVAR_MASK: shape_id_mask = 8912894; +pub type shape_id_mask = u32; #[repr(C)] pub struct rb_cvar_class_tbl_entry { + pub imemo_flags: VALUE, pub index: u32, pub global_cvar_state: rb_serial_t, pub cref: *const rb_cref_t, @@ -715,7 +661,7 @@ pub type vm_call_flag_bits = u32; #[repr(C)] pub struct rb_callinfo_kwarg { pub keyword_len: ::std::os::raw::c_int, - pub references: ::std::os::raw::c_int, + pub references: rb_atomic_t, pub keywords: __IncompleteArrayField<VALUE>, } #[repr(C)] @@ -723,8 +669,8 @@ pub struct rb_callinfo { pub flags: VALUE, pub kwarg: *const rb_callinfo_kwarg, pub mid: VALUE, - pub flag: VALUE, - pub argc: VALUE, + pub flag: ::std::os::raw::c_uint, + pub argc: ::std::os::raw::c_uint, } #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -773,8 +719,8 @@ pub const YARVINSN_putnil: ruby_vminsn_type = 17; pub const YARVINSN_putself: ruby_vminsn_type = 18; pub const YARVINSN_putobject: ruby_vminsn_type = 19; pub const YARVINSN_putspecialobject: ruby_vminsn_type = 20; -pub const YARVINSN_putstring: ruby_vminsn_type = 21; -pub const YARVINSN_putchilledstring: ruby_vminsn_type = 22; +pub const YARVINSN_dupstring: ruby_vminsn_type = 21; +pub const YARVINSN_dupchilledstring: ruby_vminsn_type = 22; pub const YARVINSN_concatstrings: ruby_vminsn_type = 23; pub const YARVINSN_anytostring: ruby_vminsn_type = 24; pub const YARVINSN_toregexp: ruby_vminsn_type = 25; @@ -828,153 +774,190 @@ pub const YARVINSN_jump: ruby_vminsn_type = 72; pub const YARVINSN_branchif: ruby_vminsn_type = 73; pub const YARVINSN_branchunless: ruby_vminsn_type = 74; pub const YARVINSN_branchnil: ruby_vminsn_type = 75; -pub const YARVINSN_once: ruby_vminsn_type = 76; -pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 77; -pub const YARVINSN_opt_plus: ruby_vminsn_type = 78; -pub const YARVINSN_opt_minus: ruby_vminsn_type = 79; -pub const YARVINSN_opt_mult: ruby_vminsn_type = 80; -pub const YARVINSN_opt_div: ruby_vminsn_type = 81; -pub const YARVINSN_opt_mod: ruby_vminsn_type = 82; -pub const YARVINSN_opt_eq: ruby_vminsn_type = 83; -pub const YARVINSN_opt_neq: ruby_vminsn_type = 84; -pub const YARVINSN_opt_lt: ruby_vminsn_type = 85; -pub const YARVINSN_opt_le: ruby_vminsn_type = 86; -pub const YARVINSN_opt_gt: ruby_vminsn_type = 87; -pub const YARVINSN_opt_ge: ruby_vminsn_type = 88; -pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 89; -pub const YARVINSN_opt_and: ruby_vminsn_type = 90; -pub const YARVINSN_opt_or: ruby_vminsn_type = 91; -pub const YARVINSN_opt_aref: ruby_vminsn_type = 92; -pub const YARVINSN_opt_aset: ruby_vminsn_type = 93; -pub const YARVINSN_opt_aset_with: ruby_vminsn_type = 94; -pub const YARVINSN_opt_aref_with: ruby_vminsn_type = 95; -pub const YARVINSN_opt_length: ruby_vminsn_type = 96; -pub const YARVINSN_opt_size: ruby_vminsn_type = 97; -pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 98; -pub const YARVINSN_opt_succ: ruby_vminsn_type = 99; -pub const YARVINSN_opt_not: ruby_vminsn_type = 100; -pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 101; -pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 102; -pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 103; -pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 104; -pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 105; -pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 106; -pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 107; -pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 108; -pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 109; -pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 110; -pub const YARVINSN_trace_nop: ruby_vminsn_type = 111; -pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 112; -pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 113; -pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 114; -pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 115; -pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 116; -pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 117; -pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 118; -pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 119; -pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 120; -pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 121; -pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 122; -pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 123; -pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 124; -pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 125; -pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 126; -pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 127; -pub const YARVINSN_trace_putnil: ruby_vminsn_type = 128; -pub const YARVINSN_trace_putself: ruby_vminsn_type = 129; -pub const YARVINSN_trace_putobject: ruby_vminsn_type = 130; -pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 131; -pub const YARVINSN_trace_putstring: ruby_vminsn_type = 132; -pub const YARVINSN_trace_putchilledstring: ruby_vminsn_type = 133; -pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 134; -pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 135; -pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 136; -pub const YARVINSN_trace_intern: ruby_vminsn_type = 137; -pub const YARVINSN_trace_newarray: ruby_vminsn_type = 138; -pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 139; -pub const YARVINSN_trace_duparray: ruby_vminsn_type = 140; -pub const YARVINSN_trace_duphash: ruby_vminsn_type = 141; -pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 142; -pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 143; -pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 144; -pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 145; -pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 146; -pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 147; -pub const YARVINSN_trace_newhash: ruby_vminsn_type = 148; -pub const YARVINSN_trace_newrange: ruby_vminsn_type = 149; -pub const YARVINSN_trace_pop: ruby_vminsn_type = 150; -pub const YARVINSN_trace_dup: ruby_vminsn_type = 151; -pub const YARVINSN_trace_dupn: ruby_vminsn_type = 152; -pub const YARVINSN_trace_swap: ruby_vminsn_type = 153; -pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 154; -pub const YARVINSN_trace_topn: ruby_vminsn_type = 155; -pub const YARVINSN_trace_setn: ruby_vminsn_type = 156; -pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 157; -pub const YARVINSN_trace_defined: ruby_vminsn_type = 158; -pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 159; -pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 160; -pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 161; -pub const YARVINSN_trace_checktype: ruby_vminsn_type = 162; -pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 163; -pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 164; -pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 165; -pub const YARVINSN_trace_send: ruby_vminsn_type = 166; -pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 167; -pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 168; -pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 169; -pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 170; -pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 171; -pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 172; -pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 173; -pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 174; -pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 175; -pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 176; -pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 177; -pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 178; -pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 179; -pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 180; -pub const YARVINSN_trace_leave: ruby_vminsn_type = 181; -pub const YARVINSN_trace_throw: ruby_vminsn_type = 182; -pub const YARVINSN_trace_jump: ruby_vminsn_type = 183; -pub const YARVINSN_trace_branchif: ruby_vminsn_type = 184; -pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 185; -pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 186; -pub const YARVINSN_trace_once: ruby_vminsn_type = 187; -pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 188; -pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 189; -pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 190; -pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 191; -pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 192; -pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 193; -pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 194; -pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 195; -pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 196; -pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 197; -pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 198; -pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 199; -pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 200; -pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 201; -pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 202; -pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 203; -pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 204; -pub const YARVINSN_trace_opt_aset_with: ruby_vminsn_type = 205; -pub const YARVINSN_trace_opt_aref_with: ruby_vminsn_type = 206; -pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 207; -pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 208; -pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 209; -pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 210; -pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 211; -pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 212; -pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 213; -pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 214; -pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 215; -pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 216; -pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 217; -pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 218; -pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 219; -pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 220; -pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 221; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 222; +pub const YARVINSN_jump_without_ints: ruby_vminsn_type = 76; +pub const YARVINSN_branchif_without_ints: ruby_vminsn_type = 77; +pub const YARVINSN_branchunless_without_ints: ruby_vminsn_type = 78; +pub const YARVINSN_branchnil_without_ints: ruby_vminsn_type = 79; +pub const YARVINSN_once: ruby_vminsn_type = 80; +pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 81; +pub const YARVINSN_opt_plus: ruby_vminsn_type = 82; +pub const YARVINSN_opt_minus: ruby_vminsn_type = 83; +pub const YARVINSN_opt_mult: ruby_vminsn_type = 84; +pub const YARVINSN_opt_div: ruby_vminsn_type = 85; +pub const YARVINSN_opt_mod: ruby_vminsn_type = 86; +pub const YARVINSN_opt_eq: ruby_vminsn_type = 87; +pub const YARVINSN_opt_neq: ruby_vminsn_type = 88; +pub const YARVINSN_opt_lt: ruby_vminsn_type = 89; +pub const YARVINSN_opt_le: ruby_vminsn_type = 90; +pub const YARVINSN_opt_gt: ruby_vminsn_type = 91; +pub const YARVINSN_opt_ge: ruby_vminsn_type = 92; +pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 93; +pub const YARVINSN_opt_and: ruby_vminsn_type = 94; +pub const YARVINSN_opt_or: ruby_vminsn_type = 95; +pub const YARVINSN_opt_aref: ruby_vminsn_type = 96; +pub const YARVINSN_opt_aset: ruby_vminsn_type = 97; +pub const YARVINSN_opt_length: ruby_vminsn_type = 98; +pub const YARVINSN_opt_size: ruby_vminsn_type = 99; +pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 100; +pub const YARVINSN_opt_succ: ruby_vminsn_type = 101; +pub const YARVINSN_opt_not: ruby_vminsn_type = 102; +pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 103; +pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 104; +pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 105; +pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 106; +pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 107; +pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 108; +pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 109; +pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 110; +pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 111; +pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 112; +pub const YARVINSN_trace_nop: ruby_vminsn_type = 113; +pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 114; +pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 115; +pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 116; +pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 117; +pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 118; +pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 119; +pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 120; +pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 121; +pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 122; +pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 123; +pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 124; +pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 125; +pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 126; +pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 127; +pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 128; +pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 129; +pub const YARVINSN_trace_putnil: ruby_vminsn_type = 130; +pub const YARVINSN_trace_putself: ruby_vminsn_type = 131; +pub const YARVINSN_trace_putobject: ruby_vminsn_type = 132; +pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 133; +pub const YARVINSN_trace_dupstring: ruby_vminsn_type = 134; +pub const YARVINSN_trace_dupchilledstring: ruby_vminsn_type = 135; +pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 136; +pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 137; +pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 138; +pub const YARVINSN_trace_intern: ruby_vminsn_type = 139; +pub const YARVINSN_trace_newarray: ruby_vminsn_type = 140; +pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 141; +pub const YARVINSN_trace_duparray: ruby_vminsn_type = 142; +pub const YARVINSN_trace_duphash: ruby_vminsn_type = 143; +pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 144; +pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 145; +pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 146; +pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 147; +pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 148; +pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 149; +pub const YARVINSN_trace_newhash: ruby_vminsn_type = 150; +pub const YARVINSN_trace_newrange: ruby_vminsn_type = 151; +pub const YARVINSN_trace_pop: ruby_vminsn_type = 152; +pub const YARVINSN_trace_dup: ruby_vminsn_type = 153; +pub const YARVINSN_trace_dupn: ruby_vminsn_type = 154; +pub const YARVINSN_trace_swap: ruby_vminsn_type = 155; +pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 156; +pub const YARVINSN_trace_topn: ruby_vminsn_type = 157; +pub const YARVINSN_trace_setn: ruby_vminsn_type = 158; +pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 159; +pub const YARVINSN_trace_defined: ruby_vminsn_type = 160; +pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 161; +pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 162; +pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 163; +pub const YARVINSN_trace_checktype: ruby_vminsn_type = 164; +pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 165; +pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 166; +pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 167; +pub const YARVINSN_trace_send: ruby_vminsn_type = 168; +pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 169; +pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 170; +pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 171; +pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 172; +pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 173; +pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 174; +pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 175; +pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 176; +pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 177; +pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 178; +pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 179; +pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 180; +pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 181; +pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 182; +pub const YARVINSN_trace_leave: ruby_vminsn_type = 183; +pub const YARVINSN_trace_throw: ruby_vminsn_type = 184; +pub const YARVINSN_trace_jump: ruby_vminsn_type = 185; +pub const YARVINSN_trace_branchif: ruby_vminsn_type = 186; +pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 187; +pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 188; +pub const YARVINSN_trace_jump_without_ints: ruby_vminsn_type = 189; +pub const YARVINSN_trace_branchif_without_ints: ruby_vminsn_type = 190; +pub const YARVINSN_trace_branchunless_without_ints: ruby_vminsn_type = 191; +pub const YARVINSN_trace_branchnil_without_ints: ruby_vminsn_type = 192; +pub const YARVINSN_trace_once: ruby_vminsn_type = 193; +pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 194; +pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 195; +pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 196; +pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 197; +pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 198; +pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 199; +pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 200; +pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 201; +pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 202; +pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 203; +pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 204; +pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 205; +pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 206; +pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 207; +pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 208; +pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 209; +pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 210; +pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 211; +pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 212; +pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 213; +pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 214; +pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 215; +pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 216; +pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 217; +pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 218; +pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 219; +pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 220; +pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 221; +pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 222; +pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 223; +pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 224; +pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 225; +pub const YARVINSN_zjit_getblockparamproxy: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_splatkw: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 248; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 249; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 250; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 251; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 252; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 253; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 254; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 255; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 256; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 257; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 258; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 259; pub type ruby_vminsn_type = u32; pub type rb_iseq_callback = ::std::option::Option< unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void), @@ -998,13 +981,22 @@ pub const DEFINED_REF: defined_type = 15; pub const DEFINED_FUNC: defined_type = 16; pub const DEFINED_CONST_FROM: defined_type = 17; pub type defined_type = u32; -pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: robject_offsets = 16; -pub const ROBJECT_OFFSET_AS_ARY: robject_offsets = 16; -pub type robject_offsets = u32; -pub const RUBY_OFFSET_RSTRING_LEN: rstring_offsets = 16; -pub type rstring_offsets = u32; -pub type rb_seq_param_keyword_struct = rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword; -pub type rb_iseq_param_keyword_struct = rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword; +pub type rb_seq_param_keyword_struct = + rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword; +pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: jit_bindgen_constants = 16; +pub const ROBJECT_OFFSET_AS_ARY: jit_bindgen_constants = 16; +pub const RCLASS_OFFSET_PRIME_FIELDS_OBJ: jit_bindgen_constants = 40; +pub const TDATA_OFFSET_FIELDS_OBJ: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_RSTRING_LEN: jit_bindgen_constants = 16; +pub const RB_SHAPE_FLAG_SHIFT: jit_bindgen_constants = 32; +pub const RUBY_OFFSET_EC_CFP: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: jit_bindgen_constants = 32; +pub const RUBY_OFFSET_EC_INTERRUPT_MASK: jit_bindgen_constants = 36; +pub const RUBY_OFFSET_EC_THREAD_PTR: jit_bindgen_constants = 48; +pub const RUBY_OFFSET_EC_RACTOR_ID: jit_bindgen_constants = 64; +pub type jit_bindgen_constants = u32; +pub type rb_iseq_param_keyword_struct = + rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword; extern "C" { pub fn ruby_xfree(ptr: *mut ::std::os::raw::c_void); pub fn rb_class_attached_object(klass: VALUE) -> VALUE; @@ -1084,7 +1076,11 @@ extern "C" { pub fn rb_obj_info_dump(obj: VALUE); pub fn rb_class_allocate_instance(klass: VALUE) -> VALUE; pub fn rb_obj_equal(obj1: VALUE, obj2: VALUE) -> VALUE; - pub fn rb_reg_new_ary(ary: VALUE, options: ::std::os::raw::c_int) -> VALUE; + pub fn rb_reg_new_from_values( + cnt: ::std::os::raw::c_long, + elements: *const VALUE, + opt: ::std::os::raw::c_int, + ) -> VALUE; pub fn rb_ary_tmp_new_from_values( arg1: VALUE, arg2: ::std::os::raw::c_long, @@ -1096,7 +1092,7 @@ extern "C" { elts: *const VALUE, ) -> VALUE; pub fn rb_vm_top_self() -> VALUE; - pub static mut rb_vm_insns_count: u64; + pub static mut rb_vm_insn_count: u64; pub fn rb_method_entry_at(obj: VALUE, id: ID) -> *const rb_method_entry_t; pub fn rb_callable_method_entry(klass: VALUE, id: ID) -> *const rb_callable_method_entry_t; pub fn rb_callable_method_entry_or_negative( @@ -1118,7 +1114,13 @@ extern "C" { pub fn rb_shape_id_offset() -> i32; pub fn rb_obj_shape_id(obj: VALUE) -> shape_id_t; pub fn rb_shape_get_iv_index(shape_id: shape_id_t, id: ID, value: *mut attr_index_t) -> bool; - pub fn rb_shape_transition_add_ivar_no_warnings(obj: VALUE, id: ID) -> shape_id_t; + pub fn rb_shape_transition_add_ivar_no_warnings( + shape_id: shape_id_t, + id: ID, + klass: VALUE, + ) -> shape_id_t; + pub fn rb_ivar_get_at(obj: VALUE, index: attr_index_t, id: ID) -> VALUE; + pub fn rb_ivar_get_at_no_ractor_check(obj: VALUE, index: attr_index_t) -> VALUE; pub fn rb_gvar_get(arg1: ID) -> VALUE; pub fn rb_gvar_set(arg1: ID, arg2: VALUE) -> VALUE; pub fn rb_ensure_iv_list_size(obj: VALUE, current_len: u32, newsize: u32); @@ -1138,11 +1140,6 @@ extern "C" { chilled: bool, ) -> VALUE; pub fn rb_to_hash_type(obj: VALUE) -> VALUE; - pub fn rb_hash_stlike_foreach( - hash: VALUE, - func: st_foreach_callback_func, - arg: st_data_t, - ) -> ::std::os::raw::c_int; pub fn rb_hash_new_with_size(size: st_index_t) -> VALUE; pub fn rb_hash_resurrect(hash: VALUE) -> VALUE; pub fn rb_hash_stlike_lookup( @@ -1168,47 +1165,22 @@ extern "C" { lines: *mut ::std::os::raw::c_int, ) -> ::std::os::raw::c_int; pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); - pub fn rb_yjit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_yjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); - pub fn rb_yjit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_yjit_array_len(a: VALUE) -> ::std::os::raw::c_long; - pub fn rb_yjit_icache_invalidate( - start: *mut ::std::os::raw::c_void, - end: *mut ::std::os::raw::c_void, - ); pub fn rb_yjit_exit_locations_dict( yjit_raw_samples: *mut VALUE, yjit_line_samples: *mut ::std::os::raw::c_int, samples_len: ::std::os::raw::c_int, ) -> VALUE; - pub fn rb_yjit_get_page_size() -> u32; - pub fn rb_yjit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_c_method_tracing_currently_enabled(ec: *const rb_execution_context_t) -> bool; pub fn rb_full_cfunc_return(ec: *mut rb_execution_context_t, return_value: VALUE); pub fn rb_iseq_get_yjit_payload(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_void; pub fn rb_iseq_set_yjit_payload(iseq: *const rb_iseq_t, payload: *mut ::std::os::raw::c_void); - pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); - pub fn rb_yjit_get_proc_ptr(procv: VALUE) -> *mut rb_proc_t; pub fn rb_get_symbol_id(namep: VALUE) -> ID; - pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE; - pub fn rb_optimized_call( - recv: *mut VALUE, - ec: *mut rb_execution_context_t, - argc: ::std::os::raw::c_int, - argv: *mut VALUE, - kw_splat: ::std::os::raw::c_int, - block_handler: VALUE, - ) -> VALUE; - pub fn rb_yjit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_yjit_builtin_function(iseq: *const rb_iseq_t) -> *const rb_builtin_function; pub fn rb_yjit_str_simple_append(str1: VALUE, str2: VALUE) -> VALUE; pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; - pub fn rb_yarv_str_eql_internal(str1: VALUE, str2: VALUE) -> VALUE; pub fn rb_str_neq_internal(str1: VALUE, str2: VALUE) -> VALUE; pub fn rb_ary_unshift_m(argc: ::std::os::raw::c_int, argv: *mut VALUE, ary: VALUE) -> VALUE; pub fn rb_yjit_rb_ary_subseq_length(ary: VALUE, beg: ::std::os::raw::c_long) -> VALUE; - pub fn rb_yjit_fix_div_fix(recv: VALUE, obj: VALUE) -> VALUE; - pub fn rb_yjit_fix_mod_fix(recv: VALUE, obj: VALUE) -> VALUE; pub fn rb_yjit_ruby2_keywords_splat_p(obj: VALUE) -> usize; pub fn rb_yjit_splat_varg_checks( sp: *mut VALUE, @@ -1220,37 +1192,32 @@ extern "C" { pub fn rb_yjit_iseq_inspect(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_char; pub fn rb_RSTRUCT_SET(st: VALUE, k: ::std::os::raw::c_int, v: VALUE); pub fn rb_ENCODING_GET(obj: VALUE) -> ::std::os::raw::c_int; - pub fn rb_yjit_multi_ractor_p() -> bool; pub fn rb_yjit_constcache_shareable(ice: *const iseq_inline_constant_cache_entry) -> bool; - pub fn rb_yjit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); pub fn rb_yjit_obj_written( old: VALUE, young: VALUE, file: *const ::std::os::raw::c_char, line: ::std::os::raw::c_int, ); - pub fn rb_yjit_vm_lock_then_barrier( - recursive_lock_level: *mut ::std::os::raw::c_uint, - file: *const ::std::os::raw::c_char, - line: ::std::os::raw::c_int, - ); - pub fn rb_yjit_vm_unlock( - recursive_lock_level: *mut ::std::os::raw::c_uint, - file: *const ::std::os::raw::c_char, - line: ::std::os::raw::c_int, - ); pub fn rb_object_shape_count() -> VALUE; - pub fn rb_yjit_shape_too_complex_p(shape_id: shape_id_t) -> bool; - pub fn rb_yjit_shape_obj_too_complex_p(obj: VALUE) -> bool; + pub fn rb_yjit_shape_obj_complex_p(obj: VALUE) -> bool; pub fn rb_yjit_shape_capacity(shape_id: shape_id_t) -> attr_index_t; pub fn rb_yjit_shape_index(shape_id: shape_id_t) -> attr_index_t; pub fn rb_yjit_sendish_sp_pops(ci: *const rb_callinfo) -> usize; pub fn rb_yjit_invokeblock_sp_pops(ci: *const rb_callinfo) -> usize; + pub fn rb_yjit_cme_ractor_serial(cme: *const rb_callable_method_entry_t) -> rb_serial_t; pub fn rb_yjit_set_exception_return( cfp: *mut rb_control_frame_t, leave_exit: *mut ::std::os::raw::c_void, leave_exception: *mut ::std::os::raw::c_void, ); + pub fn rb_vm_instruction_size() -> u32; + pub fn rb_yjit_cdhash_all_fixnum_p(cdhash: VALUE) -> bool; + pub fn rb_yjit_cdhash_lookup( + cdhash: VALUE, + key: st_data_t, + val: *mut st_data_t, + ) -> ::std::os::raw::c_int; pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_iseq_pc_at_idx(iseq: *const rb_iseq_t, insn_idx: u32) -> *mut VALUE; pub fn rb_iseq_opcode_at_pc(iseq: *const rb_iseq_t, pc: *const VALUE) -> ::std::os::raw::c_int; @@ -1280,6 +1247,17 @@ extern "C" { ) -> *mut rb_method_cfunc_t; pub fn rb_get_def_method_serial(def: *const rb_method_definition_t) -> usize; pub fn rb_get_def_original_id(def: *const rb_method_definition_t) -> ID; + pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE; + pub fn rb_jit_get_proc_ptr(procv: VALUE) -> *mut rb_proc_t; + pub fn rb_optimized_call( + recv: VALUE, + ec: *mut rb_execution_context_t, + argc: ::std::os::raw::c_int, + argv: *mut VALUE, + kw_splat: ::std::os::raw::c_int, + block_handler: VALUE, + ) -> VALUE; + pub fn rb_jit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_get_mct_argc(mct: *const rb_method_cfunc_t) -> ::std::os::raw::c_int; pub fn rb_get_mct_func(mct: *const rb_method_cfunc_t) -> *mut ::std::os::raw::c_void; pub fn rb_get_def_iseq_ptr(def: *mut rb_method_definition_t) -> *const rb_iseq_t; @@ -1328,6 +1306,34 @@ extern "C" { pub fn rb_IMEMO_TYPE_P(imemo: VALUE, imemo_type: imemo_type) -> ::std::os::raw::c_int; pub fn rb_assert_cme_handle(handle: VALUE); pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE; + pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long; pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE); pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE); + pub fn rb_jit_shape_complex_p(shape_id: shape_id_t) -> bool; + pub fn rb_jit_multi_ractor_p() -> bool; + pub fn rb_jit_vm_lock_then_barrier( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_jit_vm_unlock( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_get_page_size() -> u32; + pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8; + pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); + pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); + pub fn rb_jit_fix_mod_fix(recv: VALUE, obj: VALUE) -> VALUE; + pub fn rb_jit_fix_div_fix(recv: VALUE, obj: VALUE) -> VALUE; + pub fn rb_yarv_str_eql_internal(str1: VALUE, str2: VALUE) -> VALUE; + pub fn rb_jit_str_concat_codepoint(str_: VALUE, codepoint: VALUE); } diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs index 6ae1342ce3..f3bb91ddc1 100644 --- a/yjit/src/invariants.rs +++ b/yjit/src/invariants.rs @@ -168,8 +168,8 @@ pub fn track_no_ep_escape_assumption(uninit_block: BlockRef, iseq: IseqPtr) { .insert(uninit_block); } -/// Returns true if a given ISEQ has previously escaped an environment. -pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool { +/// Returns true if a given ISEQ has escaped an environment since YJIT boot. +pub fn seen_escaped_env(iseq: IseqPtr) -> bool { Invariants::get_instance() .no_ep_escape_iseqs .get(&iseq) @@ -206,7 +206,7 @@ pub fn assume_method_basic_definition( /// Tracks that a block is assuming it is operating in single-ractor mode. #[must_use] pub fn assume_single_ractor_mode(jit: &mut JITState, asm: &mut Assembler) -> bool { - if unsafe { rb_yjit_multi_ractor_p() } { + if unsafe { rb_jit_multi_ractor_p() } { false } else { if jit_ensure_block_entry_exit(jit, asm).is_none() { @@ -495,7 +495,7 @@ pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, ins return; }; - if !unsafe { (*(*ic).entry).ic_cref }.is_null() || unsafe { rb_yjit_multi_ractor_p() } { + if !unsafe { (*(*ic).entry).ic_cref }.is_null() || unsafe { rb_jit_multi_ractor_p() } { // We can't generate code in these situations, so no need to invalidate. // See gen_opt_getinlinecache. return; @@ -642,7 +642,7 @@ pub extern "C" fn rb_yjit_tracing_invalidate_all() { if on_stack_iseqs.contains(&iseq) { // This ISEQ is running, so we can't free blocks immediately for block in blocks { - delayed_deallocation(block); + payload.delayed_deallocation(block); } payload.dead_blocks.shrink_to_fit(); } else { diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index ba84b7a549..b23cd91618 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -1,9 +1,8 @@ //! Everything related to the collection of runtime stats in YJIT //! See the --yjit-stats command-line option -use std::alloc::{GlobalAlloc, Layout, System}; use std::ptr::addr_of_mut; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::Ordering; use std::time::Instant; use std::collections::HashMap; @@ -12,6 +11,10 @@ use crate::cruby::*; use crate::options::*; use crate::yjit::{yjit_enabled_p, YJIT_INIT_TIME}; +#[cfg(feature = "stats_allocator")] +#[path = "../../jit/src/lib.rs"] +mod jit; + /// Running total of how many ISeqs are in the system. #[no_mangle] pub static mut rb_yjit_live_iseq_count: u64 = 0; @@ -20,43 +23,9 @@ pub static mut rb_yjit_live_iseq_count: u64 = 0; #[no_mangle] pub static mut rb_yjit_iseq_alloc_count: u64 = 0; -/// A middleware to count Rust-allocated bytes as yjit_alloc_size. -#[global_allocator] -static GLOBAL_ALLOCATOR: StatsAlloc = StatsAlloc { alloc_size: AtomicUsize::new(0) }; - -pub struct StatsAlloc { - alloc_size: AtomicUsize, -} - -unsafe impl GlobalAlloc for StatsAlloc { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst); - System.alloc(layout) - } - - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - self.alloc_size.fetch_sub(layout.size(), Ordering::SeqCst); - System.dealloc(ptr, layout) - } - - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - self.alloc_size.fetch_add(layout.size(), Ordering::SeqCst); - System.alloc_zeroed(layout) - } - - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - if new_size > layout.size() { - self.alloc_size.fetch_add(new_size - layout.size(), Ordering::SeqCst); - } else if new_size < layout.size() { - self.alloc_size.fetch_sub(layout.size() - new_size, Ordering::SeqCst); - } - System.realloc(ptr, layout, new_size) - } -} - /// The number of bytes YJIT has allocated on the Rust heap. pub fn yjit_alloc_size() -> usize { - GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst) + jit::GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst) } /// Mapping of C function / ISEQ name to integer indices @@ -121,7 +90,9 @@ pub extern "C" fn incr_iseq_counter(idx: usize) { iseq_call_count[idx] += 1; } -// YJIT exit counts for each instruction type +/// YJIT exit counts for each instruction type. +/// Note that `VM_INSTRUCTION_SIZE` is an upper bound and the actual number +/// of VM opcodes may be different in the build. See [`rb_vm_instruction_size()`] const VM_INSTRUCTION_SIZE_USIZE: usize = VM_INSTRUCTION_SIZE as usize; static mut EXIT_OP_COUNT: [u64; VM_INSTRUCTION_SIZE_USIZE] = [0; VM_INSTRUCTION_SIZE_USIZE]; @@ -277,6 +248,7 @@ pub const DEFAULT_COUNTERS: &'static [Counter] = &[ Counter::compiled_blockid_count, Counter::compiled_block_count, Counter::deleted_defer_block_count, + Counter::exceptional_entry_escaped_env, Counter::compiled_branch_count, Counter::compile_time_ns, Counter::compilation_failure, @@ -349,7 +321,6 @@ macro_rules! ptr_to_counter { } }; } -pub(crate) use ptr_to_counter; // Declare all the counters we track make_counters! { @@ -384,6 +355,7 @@ make_counters! { send_iseq_arity_error, send_iseq_block_arg_type, send_iseq_clobbering_block_arg, + send_iseq_block_arg_gc_unsafe, send_iseq_complex_discard_extras, send_iseq_leaf_builtin_block_arg_block_param, send_iseq_kw_splat_non_nil, @@ -431,6 +403,7 @@ make_counters! { invokesuper_megamorphic, invokesuper_no_cme, invokesuper_no_me, + invokesuper_bmethod_zsuper, invokesuper_not_iseq_or_cfunc, invokesuper_refinement, invokesuper_singleton_class, @@ -518,9 +491,7 @@ make_counters! { opt_aset_not_array, opt_aset_not_fixnum, opt_aset_not_hash, - - opt_aref_with_qundef, - opt_aset_with_qundef, + opt_aset_frozen, opt_case_dispatch_megamorphic, @@ -531,6 +502,7 @@ make_counters! { expandarray_postarg, expandarray_not_array, expandarray_to_ary, + expandarray_method_missing, expandarray_chain_max_depth, // getblockparam @@ -630,6 +602,8 @@ make_counters! { iseq_stack_too_large, iseq_too_long, + exceptional_entry_escaped_env, + temp_reg_opnd, temp_mem_opnd, temp_spill, @@ -792,8 +766,8 @@ fn rb_yjit_gen_stats_dict(key: VALUE) -> VALUE { set_stat_usize!(hash, "context_cache_bytes", crate::core::CTX_ENCODE_CACHE_BYTES + crate::core::CTX_DECODE_CACHE_BYTES); // VM instructions count - if rb_vm_insns_count > 0 { - set_stat_usize!(hash, "vm_insns_count", rb_vm_insns_count as usize); + if rb_vm_insn_count > 0 { + set_stat_usize!(hash, "vm_insns_count", rb_vm_insn_count as usize); } set_stat_usize!(hash, "live_iseq_count", rb_yjit_live_iseq_count as usize); @@ -838,7 +812,8 @@ fn rb_yjit_gen_stats_dict(key: VALUE) -> VALUE { // For each entry in exit_op_count, add a stats entry with key "exit_INSTRUCTION_NAME" // and the value is the count of side exits for that instruction. - for op_idx in 0..VM_INSTRUCTION_SIZE_USIZE { + use crate::utils::IntoUsize; + for op_idx in 0..rb_vm_instruction_size().as_usize() { let op_name = insn_name(op_idx); let key_string = "exit_".to_owned() + &op_name; let count = EXIT_OP_COUNT[op_idx]; @@ -864,8 +839,8 @@ fn rb_yjit_gen_stats_dict(key: VALUE) -> VALUE { set_stat_double!(hash, "avg_len_in_yjit", avg_len_in_yjit); // Proportion of instructions that retire in YJIT - if rb_vm_insns_count > 0 { - let total_insns_count = retired_in_yjit + rb_vm_insns_count; + if rb_vm_insn_count > 0 { + let total_insns_count = retired_in_yjit + rb_vm_insn_count; set_stat_usize!(hash, "total_insns_count", total_insns_count as usize); let ratio_in_yjit: f64 = 100.0 * retired_in_yjit as f64 / total_insns_count as f64; @@ -927,7 +902,7 @@ fn rb_yjit_gen_stats_dict(key: VALUE) -> VALUE { /// and line samples. Their length should be the same, however the data stored in /// them is different. #[no_mangle] -pub extern "C" fn rb_yjit_record_exit_stack(_exit_pc: *const VALUE) +pub extern "C" fn rb_yjit_record_exit_stack(exit_pc: *const VALUE) { // Return if YJIT is not enabled if !yjit_enabled_p() { @@ -951,10 +926,11 @@ pub extern "C" fn rb_yjit_record_exit_stack(_exit_pc: *const VALUE) // rb_vm_insn_addr2opcode won't work in cargo test --all-features // because it's a C function. Without insn call, this function is useless // so wrap the whole thing in a not test check. + let _ = exit_pc; #[cfg(not(test))] { // Get the opcode from the encoded insn handler at this PC - let insn = unsafe { rb_vm_insn_addr2opcode((*_exit_pc).as_ptr()) }; + let insn = unsafe { rb_vm_insn_addr2opcode((*exit_pc).as_ptr()) }; // Use the same buffer size as Stackprof. const BUFF_LEN: usize = 2048; diff --git a/yjit/src/utils.rs b/yjit/src/utils.rs index 8c4133546d..251628fabf 100644 --- a/yjit/src/utils.rs +++ b/yjit/src/utils.rs @@ -92,10 +92,7 @@ pub fn ruby_str_to_rust(v: VALUE) -> String { let str_ptr = unsafe { rb_RSTRING_PTR(v) } as *mut u8; let str_len: usize = unsafe { rb_RSTRING_LEN(v) }.try_into().unwrap(); let str_slice: &[u8] = unsafe { slice::from_raw_parts(str_ptr, str_len) }; - match String::from_utf8(str_slice.to_vec()) { - Ok(utf8) => utf8, - Err(_) => String::new(), - } + String::from_utf8(str_slice.to_vec()).unwrap_or_default() } // Location is the file defining the method, colon, method name. @@ -163,8 +160,6 @@ pub fn print_int(asm: &mut Assembler, opnd: Opnd) { } } - asm.cpush_all(); - let argument = match opnd { Opnd::Mem(_) | Opnd::Reg(_) | Opnd::InsnOut { .. } => { // Sign-extend the value if necessary @@ -179,7 +174,6 @@ pub fn print_int(asm: &mut Assembler, opnd: Opnd) { }; asm.ccall(print_int_fn as *const u8, vec![argument]); - asm.cpop_all(); } /// Generate code to print a pointer @@ -192,9 +186,7 @@ pub fn print_ptr(asm: &mut Assembler, opnd: Opnd) { assert!(opnd.rm_num_bits() == 64); - asm.cpush_all(); asm.ccall(print_ptr_fn as *const u8, vec![opnd]); - asm.cpop_all(); } /// Generate code to print a value @@ -207,9 +199,7 @@ pub fn print_value(asm: &mut Assembler, opnd: Opnd) { assert!(matches!(opnd, Opnd::Value(_))); - asm.cpush_all(); asm.ccall(print_value_fn as *const u8, vec![opnd]); - asm.cpop_all(); } /// Generate code to print constant string to stdout @@ -224,7 +214,6 @@ pub fn print_str(asm: &mut Assembler, str: &str) { } } - asm.cpush_all(); let string_data = asm.new_label("string_data"); let after_string = asm.new_label("after_string"); @@ -236,8 +225,6 @@ pub fn print_str(asm: &mut Assembler, str: &str) { let opnd = asm.lea_jump_target(string_data); asm.ccall(print_str_cfun as *const u8, vec![opnd, Opnd::UImm(str.len() as u64)]); - - asm.cpop_all(); } pub fn stdout_supports_colors() -> bool { diff --git a/yjit/src/virtualmem.rs b/yjit/src/virtualmem.rs index aa6d21f210..9126cf300e 100644 --- a/yjit/src/virtualmem.rs +++ b/yjit/src/virtualmem.rs @@ -7,6 +7,9 @@ use std::{cell::RefCell, ptr::NonNull}; use crate::{backend::ir::Target, stats::yjit_alloc_size, utils::IntoUsize}; +#[cfg(test)] +use crate::options::get_option; + #[cfg(not(test))] pub type VirtualMem = VirtualMemory<sys::SystemAllocator>; @@ -43,7 +46,7 @@ pub struct VirtualMemory<A: Allocator> { /// Mutable parts of [`VirtualMemory`]. pub struct VirtualMemoryMut<A: Allocator> { /// Number of bytes that have we have allocated physical memory for starting at - /// [Self::region_start]. + /// [VirtualMemory::region_start]. mapped_region_bytes: usize, /// Keep track of the address of the last written to page. @@ -316,15 +319,15 @@ mod sys { impl super::Allocator for SystemAllocator { fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_yjit_mark_writable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) } } fn mark_executable(&mut self, ptr: *const u8, size: u32) { - unsafe { rb_yjit_mark_executable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) } } fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_yjit_mark_unused(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) } } } } @@ -411,7 +414,7 @@ pub mod tests { PAGE_SIZE.try_into().unwrap(), NonNull::new(mem_start as *mut u8).unwrap(), mem_size, - 128 * 1024 * 1024, + get_option!(mem_size), ) } diff --git a/yjit/src/yjit.rs b/yjit/src/yjit.rs index 517a0daae5..380544ba33 100644 --- a/yjit/src/yjit.rs +++ b/yjit/src/yjit.rs @@ -37,12 +37,17 @@ pub fn yjit_enabled_p() -> bool { unsafe { rb_yjit_enabled_p } } -/// This function is called from C code +/// Register specialized codegen for builtin C method entries. +/// Must be called at boot before ruby_init_prelude() since the prelude +/// could redefine core methods (e.g. Kernel.prepend via bundler). #[no_mangle] -pub extern "C" fn rb_yjit_init(yjit_enabled: bool) { - // Register the method codegen functions. This must be done at boot. +pub extern "C" fn rb_yjit_init_builtin_cmes() { yjit_reg_method_codegen_fns(); +} +/// This function is called from C code +#[no_mangle] +pub extern "C" fn rb_yjit_init(yjit_enabled: bool) { // If --yjit-disable, yjit_init() will not be called until RubyVM::YJIT.enable. if yjit_enabled { yjit_init(); @@ -157,6 +162,13 @@ pub extern "C" fn rb_yjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr, jit_exc return std::ptr::null(); } + // In case of exceptional entry, reject escaped environment. + // This allows us to use the fact that new frames generally start with an on-stack environment. + if jit_exception && unsafe { cfp_env_has_escaped(get_ec_cfp(ec)) } { + incr_counter!(exceptional_entry_escaped_env); + return std::ptr::null(); + } + // If a custom call threshold was not specified on the command-line and // this is a large application (has very many ISEQs), switch to // using the call threshold for large applications after this entry point |
