diff options
Diffstat (limited to 'yjit/src/codegen.rs')
| -rw-r--r-- | yjit/src/codegen.rs | 2842 |
1 files changed, 1754 insertions, 1088 deletions
diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index d86f0d1955..0fbca85716 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -3,6 +3,7 @@ use crate::asm::*; use crate::backend::ir::*; +use crate::backend::current::TEMP_REGS; use crate::core::*; use crate::cruby::*; use crate::invariants::*; @@ -30,7 +31,6 @@ pub use crate::virtualmem::CodePtr; /// Status returned by code generation functions #[derive(PartialEq, Debug)] enum CodegenStatus { - SkipNextInsn, KeepCompiling, EndBlock, } @@ -39,12 +39,11 @@ enum CodegenStatus { type InsnGenFn = fn( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus>; /// Ephemeral code generation state. -/// Represents a [core::Block] while we build it. -pub struct JITState { +/// Represents a [crate::core::Block] while we build it. +pub struct JITState<'a> { /// Instruction sequence for the compiling block pub iseq: IseqPtr, @@ -73,6 +72,10 @@ pub struct JITState { /// This allows us to peek at run-time values ec: EcPtr, + /// The code block used for stubs, exits, and other code that are + /// not on the hot path. + outlined_code_block: &'a mut OutlinedCb, + /// The outgoing branches the block will have pub pending_outgoing: Vec<PendingBranchRef>, @@ -112,10 +115,19 @@ pub struct JITState { /// Stack of symbol names for --yjit-perf perf_stack: Vec<String>, + + /// When true, this block is the first block compiled by gen_block_series(). + first_block: bool, + + /// A killswitch for bailing out of compilation. Used in rare situations where we need to fail + /// compilation deep in the stack (e.g. codegen failed for some jump target, but not due to + /// OOM). Because these situations are so rare it's not worth it to check and propogate at each + /// site. Instead, we check this once at the end. + block_abandoned: bool, } -impl JITState { - pub fn new(blockid: BlockId, starting_ctx: Context, output_ptr: CodePtr, ec: EcPtr) -> Self { +impl<'a> JITState<'a> { + pub fn new(blockid: BlockId, starting_ctx: Context, output_ptr: CodePtr, ec: EcPtr, ocb: &'a mut OutlinedCb, first_block: bool) -> Self { JITState { iseq: blockid.iseq, starting_insn_idx: blockid.idx, @@ -127,6 +139,7 @@ impl JITState { stack_size_for_pc: starting_ctx.get_stack_size(), pending_outgoing: vec![], ec, + outlined_code_block: ocb, record_boundary_patch_point: false, block_entry_exit: None, method_lookup_assumptions: vec![], @@ -137,6 +150,8 @@ impl JITState { block_assumes_single_ractor: false, perf_map: Rc::default(), perf_stack: vec![], + first_block, + block_abandoned: false, } } @@ -144,15 +159,15 @@ impl JITState { self.insn_idx } - pub fn get_iseq(self: &JITState) -> IseqPtr { + pub fn get_iseq(&self) -> IseqPtr { self.iseq } - pub fn get_opcode(self: &JITState) -> usize { + pub fn get_opcode(&self) -> usize { self.opcode } - pub fn get_pc(self: &JITState) -> *mut VALUE { + pub fn get_pc(&self) -> *mut VALUE { self.pc } @@ -175,6 +190,50 @@ impl JITState { unsafe { *(self.pc.offset(arg_idx + 1)) } } + /// Get [Self::outlined_code_block] + pub fn get_ocb(&mut self) -> &mut OutlinedCb { + self.outlined_code_block + } + + /// Leave a code stub to re-enter the compiler at runtime when the compiling program point is + /// reached. Should always be used in tail position like `return jit.defer_compilation(asm);`. + #[must_use] + fn defer_compilation(&mut self, asm: &mut Assembler) -> Option<CodegenStatus> { + if crate::core::defer_compilation(self, asm).is_err() { + // If we can't leave a stub, the block isn't usable and we have to bail. + self.block_abandoned = true; + } + Some(EndBlock) + } + + /// Generate a branch with either end possibly stubbed out + fn gen_branch( + &mut self, + asm: &mut Assembler, + target0: BlockId, + ctx0: &Context, + target1: Option<BlockId>, + ctx1: Option<&Context>, + gen_fn: BranchGenFn, + ) { + if crate::core::gen_branch(self, asm, target0, ctx0, target1, ctx1, gen_fn).is_none() { + // If we can't meet the request for a branch, the code is + // essentially corrupt and we have to discard the block. + self.block_abandoned = true; + } + } + + /// Wrapper for [self::gen_outlined_exit] with error handling. + fn gen_outlined_exit(&mut self, exit_pc: *mut VALUE, ctx: &Context) -> Option<CodePtr> { + let result = gen_outlined_exit(exit_pc, self.num_locals(), ctx, self.get_ocb()); + if result.is_none() { + // When we can't have the exits, the code is incomplete and we have to bail. + self.block_abandoned = true; + } + + result + } + /// Return true if the current ISEQ could escape an environment. /// /// As of vm_push_frame(), EP is always equal to BP. However, after pushing @@ -197,9 +256,23 @@ impl JITState { self.insn_idx + insn_len(self.get_opcode()) as u16 } - // Check if we are compiling the instruction at the stub PC + /// Get the index of the next instruction of the next instruction + fn next_next_insn_idx(&self) -> u16 { + let next_pc = unsafe { rb_iseq_pc_at_idx(self.iseq, self.next_insn_idx().into()) }; + let next_opcode: usize = unsafe { rb_iseq_opcode_at_pc(self.iseq, next_pc) }.try_into().unwrap(); + self.next_insn_idx() + insn_len(next_opcode) as u16 + } + + // Check if we are compiling the instruction at the stub PC with the target Context // Meaning we are compiling the instruction that is next to execute - pub fn at_current_insn(&self) -> bool { + pub fn at_compile_target(&self) -> bool { + // If this is not the first block compiled by gen_block_series(), + // it might be compiling the same block again with a different Context. + // In that case, it should defer_compilation() and inspect the stack there. + if !self.first_block { + return false; + } + let ec_pc: *mut VALUE = unsafe { get_cfp_pc(self.get_cfp()) }; ec_pc == self.pc } @@ -207,7 +280,7 @@ impl JITState { // Peek at the nth topmost value on the Ruby stack. // Returns the topmost value when n == 0. pub fn peek_at_stack(&self, ctx: &Context, n: isize) -> VALUE { - assert!(self.at_current_insn()); + assert!(self.at_compile_target()); assert!(n < ctx.get_stack_size() as isize); // Note: this does not account for ctx->sp_offset because @@ -226,7 +299,7 @@ impl JITState { } fn peek_at_local(&self, n: i32) -> VALUE { - assert!(self.at_current_insn()); + assert!(self.at_compile_target()); let local_table_size: isize = unsafe { get_iseq_body_local_table_size(self.iseq) } .try_into() @@ -242,7 +315,7 @@ impl JITState { } fn peek_at_block_handler(&self, level: u32) -> VALUE { - assert!(self.at_current_insn()); + assert!(self.at_compile_target()); unsafe { let ep = get_cfp_ep_level(self.get_cfp(), level); @@ -253,7 +326,6 @@ impl JITState { pub fn assume_expected_cfunc( &mut self, asm: &mut Assembler, - ocb: &mut OutlinedCb, class: VALUE, method: ID, cfunc: *mut c_void, @@ -272,13 +344,13 @@ impl JITState { return false; } - self.assume_method_lookup_stable(asm, ocb, cme); + self.assume_method_lookup_stable(asm, cme); true } - pub fn assume_method_lookup_stable(&mut self, asm: &mut Assembler, ocb: &mut OutlinedCb, cme: CmePtr) -> Option<()> { - jit_ensure_block_entry_exit(self, asm, ocb)?; + pub fn assume_method_lookup_stable(&mut self, asm: &mut Assembler, cme: CmePtr) -> Option<()> { + jit_ensure_block_entry_exit(self, asm)?; self.method_lookup_assumptions.push(cme); Some(()) @@ -287,8 +359,8 @@ impl JITState { /// Assume that objects of a given class will have no singleton class. /// Return true if there has been no such singleton class since boot /// and we can safely invalidate it. - pub fn assume_no_singleton_class(&mut self, asm: &mut Assembler, ocb: &mut OutlinedCb, klass: VALUE) -> bool { - if jit_ensure_block_entry_exit(self, asm, ocb).is_none() { + pub fn assume_no_singleton_class(&mut self, asm: &mut Assembler, klass: VALUE) -> bool { + if jit_ensure_block_entry_exit(self, asm).is_none() { return false; // out of space, give up } if has_singleton_class_of(klass) { @@ -300,8 +372,8 @@ impl JITState { /// Assume that base pointer is equal to environment pointer in the current ISEQ. /// Return true if it's safe to assume so. - fn assume_no_ep_escape(&mut self, asm: &mut Assembler, ocb: &mut OutlinedCb) -> bool { - if jit_ensure_block_entry_exit(self, asm, ocb).is_none() { + fn assume_no_ep_escape(&mut self, asm: &mut Assembler) -> bool { + if jit_ensure_block_entry_exit(self, asm).is_none() { return false; // out of space, give up } if self.escapes_ep() { @@ -315,8 +387,8 @@ impl JITState { unsafe { get_ec_cfp(self.ec) } } - pub fn assume_stable_constant_names(&mut self, asm: &mut Assembler, ocb: &mut OutlinedCb, id: *const ID) -> Option<()> { - jit_ensure_block_entry_exit(self, asm, ocb)?; + pub fn assume_stable_constant_names(&mut self, asm: &mut Assembler, id: *const ID) -> Option<()> { + jit_ensure_block_entry_exit(self, asm)?; self.stable_constant_names_assumption = Some(id); Some(()) @@ -366,7 +438,7 @@ impl JITState { fn flush_perf_symbols(&self, cb: &CodeBlock) { assert_eq!(0, self.perf_stack.len()); let path = format!("/tmp/perf-{}.map", std::process::id()); - let mut f = std::fs::File::options().create(true).append(true).open(path).unwrap(); + let mut f = std::io::BufWriter::new(std::fs::File::options().create(true).append(true).open(path).unwrap()); for sym in self.perf_map.borrow().iter() { if let (start, Some(end), name) = sym { // In case the code straddles two pages, part of it belongs to the symbol. @@ -388,6 +460,11 @@ impl JITState { _ => false, } } + + /// Return the number of locals in the current ISEQ + pub fn num_locals(&self) -> u32 { + unsafe { get_iseq_body_local_table_size(self.iseq) } + } } /// Macro to call jit.perf_symbol_push() without evaluating arguments when @@ -427,6 +504,7 @@ macro_rules! perf_call { } use crate::codegen::JCCKinds::*; +use crate::log::Log; #[allow(non_camel_case_types, unused)] pub enum JCCKinds { @@ -441,8 +519,31 @@ pub enum JCCKinds { JCC_JO_MUL, } +/// Generate code to increment a given counter. With --yjit-trace-exits=counter, +/// the counter is traced when it's incremented by this function. #[inline(always)] -fn gen_counter_incr(asm: &mut Assembler, counter: Counter) { +fn gen_counter_incr(jit: &JITState, asm: &mut Assembler, counter: Counter) { + gen_counter_incr_with_pc(asm, counter, jit.pc); +} + +/// Same as gen_counter_incr(), but takes PC isntead of JITState. +#[inline(always)] +fn gen_counter_incr_with_pc(asm: &mut Assembler, counter: Counter, pc: *mut VALUE) { + gen_counter_incr_without_pc(asm, counter); + + // Trace a counter if --yjit-trace-exits=counter is given. + // TraceExits::All is handled by gen_exit(). + if get_option!(trace_exits) == Some(TraceExits::Counter(counter)) { + with_caller_saved_temp_regs(asm, |asm| { + asm.ccall(rb_yjit_record_exit_stack as *const u8, vec![Opnd::const_ptr(pc as *const u8)]); + }); + } +} + +/// Generate code to increment a given counter. Not traced by --yjit-trace-exits=counter +/// unlike gen_counter_incr() or gen_counter_incr_with_pc(). +#[inline(always)] +fn gen_counter_incr_without_pc(asm: &mut Assembler, counter: Counter) { // Assert that default counters are not incremented by generated code as this would impact performance assert!(!DEFAULT_COUNTERS.contains(&counter), "gen_counter_incr incremented {:?}", counter); @@ -614,7 +715,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { } // Only able to check types when at current insn - assert!(jit.at_current_insn()); + assert!(jit.at_compile_target()); let self_val = jit.peek_at_self(); let self_val_type = Type::from(self_val); @@ -632,7 +733,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { } // Verify stack operand types - let top_idx = cmp::min(ctx.get_stack_size(), MAX_TEMP_TYPES as u8); + let top_idx = cmp::min(ctx.get_stack_size(), MAX_CTX_TEMPS as u8); for i in 0..top_idx { let learned_mapping = ctx.get_opnd_mapping(StackOpnd(i)); let learned_type = ctx.get_opnd_type(StackOpnd(i)); @@ -641,8 +742,8 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { let stack_val = jit.peek_at_stack(ctx, i as isize); let val_type = Type::from(stack_val); - match learned_mapping.get_kind() { - TempMappingKind::MapToSelf => { + match learned_mapping { + TempMapping::MapToSelf => { if self_val != stack_val { panic!( "verify_ctx: stack value was mapped to self, but values did not match!\n stack: {}\n self: {}", @@ -651,8 +752,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { ); } } - TempMappingKind::MapToLocal => { - let local_idx: u8 = learned_mapping.get_local_idx(); + TempMapping::MapToLocal(local_idx) => { let local_val = jit.peek_at_local(local_idx.into()); if local_val != stack_val { panic!( @@ -663,7 +763,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { ); } } - TempMappingKind::MapToStack => {} + TempMapping::MapToStack(_) => {} } // If the actual type differs from the learned type @@ -679,7 +779,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { // Verify local variable types let local_table_size = unsafe { get_iseq_body_local_table_size(jit.iseq) }; - let top_idx: usize = cmp::min(local_table_size as usize, MAX_TEMP_TYPES); + let top_idx: usize = cmp::min(local_table_size as usize, MAX_CTX_TEMPS); for i in 0..top_idx { let learned_type = ctx.get_local_type(i); let learned_type = relax_type_with_singleton_class_assumption(learned_type); @@ -703,9 +803,9 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { // interpreter state. fn gen_stub_exit(ocb: &mut OutlinedCb) -> Option<CodePtr> { let ocb = ocb.unwrap(); - let mut asm = Assembler::new(); + let mut asm = Assembler::new_without_iseq(); - gen_counter_incr(&mut asm, Counter::exit_from_branch_stub); + gen_counter_incr_without_pc(&mut asm, Counter::exit_from_branch_stub); asm_comment!(asm, "exit from branch stub"); asm.cpop_into(SP); @@ -721,11 +821,11 @@ fn gen_stub_exit(ocb: &mut OutlinedCb) -> Option<CodePtr> { /// Generate an exit to return to the interpreter fn gen_exit(exit_pc: *mut VALUE, asm: &mut Assembler) { - #[cfg(all(feature = "disasm", not(test)))] - { + #[cfg(not(test))] + asm_comment!(asm, "exit to interpreter on {}", { let opcode = unsafe { rb_vm_insn_addr2opcode((*exit_pc).as_ptr()) }; - asm_comment!(asm, "exit to interpreter on {}", insn_name(opcode as usize)); - } + insn_name(opcode as usize) + }); if asm.ctx.is_return_landing() { asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); @@ -734,7 +834,7 @@ fn gen_exit(exit_pc: *mut VALUE, asm: &mut Assembler) { } // Spill stack temps before returning to the interpreter - asm.spill_temps(); + asm.spill_regs(); // Generate the code to exit to the interpreters // Write the adjusted SP back into the CFP @@ -790,11 +890,15 @@ fn gen_exit(exit_pc: *mut VALUE, asm: &mut Assembler) { /// moment, so there is one unique side exit for each context. Note that /// it's incorrect to jump to the side exit after any ctx stack push operations /// since they change the logic required for reconstructing interpreter state. -pub fn gen_outlined_exit(exit_pc: *mut VALUE, ctx: &Context, ocb: &mut OutlinedCb) -> Option<CodePtr> { +/// +/// If you're in [the codegen module][self], use [JITState::gen_outlined_exit] +/// instead of calling this directly. +#[must_use] +pub fn gen_outlined_exit(exit_pc: *mut VALUE, num_locals: u32, ctx: &Context, ocb: &mut OutlinedCb) -> Option<CodePtr> { let mut cb = ocb.unwrap(); - let mut asm = Assembler::new(); + let mut asm = Assembler::new(num_locals); asm.ctx = *ctx; - asm.set_reg_temps(ctx.get_reg_temps()); + asm.set_reg_mapping(ctx.get_reg_mapping()); gen_exit(exit_pc, &mut asm); @@ -812,18 +916,10 @@ pub fn gen_counted_exit(exit_pc: *mut VALUE, side_exit: CodePtr, ocb: &mut Outli None => return Some(side_exit), }; - let mut asm = Assembler::new(); + let mut asm = Assembler::new_without_iseq(); // Increment a counter - gen_counter_incr(&mut asm, counter); - - // Trace a counted exit if --yjit-trace-exits=counter is given. - // TraceExits::All is handled by gen_exit(). - if get_option!(trace_exits) == Some(TraceExits::CountedExit(counter)) { - with_caller_saved_temp_regs(&mut asm, |asm| { - asm.ccall(rb_yjit_record_exit_stack as *const u8, vec![Opnd::const_ptr(exit_pc as *const u8)]); - }); - } + gen_counter_incr_with_pc(&mut asm, counter, exit_pc); // Jump to the existing side exit asm.jmp(Target::CodePtr(side_exit)); @@ -847,7 +943,7 @@ fn with_caller_saved_temp_regs<F, R>(asm: &mut Assembler, block: F) -> R where F // Ensure that there is an exit for the start of the block being compiled. // Block invalidation uses this exit. #[must_use] -pub fn jit_ensure_block_entry_exit(jit: &mut JITState, asm: &mut Assembler, ocb: &mut OutlinedCb) -> Option<()> { +pub fn jit_ensure_block_entry_exit(jit: &mut JITState, asm: &mut Assembler) -> Option<()> { if jit.block_entry_exit.is_some() { return Some(()); } @@ -858,11 +954,11 @@ pub fn jit_ensure_block_entry_exit(jit: &mut JITState, asm: &mut Assembler, ocb: if jit.insn_idx == jit.starting_insn_idx { // Generate the exit with the cache in Assembler. let side_exit_context = SideExitContext::new(jit.pc, *block_starting_context); - let entry_exit = asm.get_side_exit(&side_exit_context, None, ocb); + let entry_exit = asm.get_side_exit(&side_exit_context, None, jit.get_ocb()); jit.block_entry_exit = Some(entry_exit?); } else { let block_entry_pc = unsafe { rb_iseq_pc_at_idx(jit.iseq, jit.starting_insn_idx.into()) }; - jit.block_entry_exit = Some(gen_outlined_exit(block_entry_pc, block_starting_context, ocb)?); + jit.block_entry_exit = Some(jit.gen_outlined_exit(block_entry_pc, block_starting_context)?); } Some(()) @@ -871,7 +967,7 @@ pub fn jit_ensure_block_entry_exit(jit: &mut JITState, asm: &mut Assembler, ocb: // Landing code for when c_return tracing is enabled. See full_cfunc_return(). fn gen_full_cfunc_return(ocb: &mut OutlinedCb) -> Option<CodePtr> { let ocb = ocb.unwrap(); - let mut asm = Assembler::new(); + let mut asm = Assembler::new_without_iseq(); // This chunk of code expects REG_EC to be filled properly and // RAX to contain the return value of the C method. @@ -883,7 +979,7 @@ fn gen_full_cfunc_return(ocb: &mut OutlinedCb) -> Option<CodePtr> { ); // Count the exit - gen_counter_incr(&mut asm, Counter::traced_cfunc_return); + gen_counter_incr_without_pc(&mut asm, Counter::traced_cfunc_return); // Return to the interpreter asm.cpop_into(SP); @@ -901,14 +997,14 @@ fn gen_full_cfunc_return(ocb: &mut OutlinedCb) -> Option<CodePtr> { /// This is used by gen_leave() and gen_entry_prologue() fn gen_leave_exit(ocb: &mut OutlinedCb) -> Option<CodePtr> { let ocb = ocb.unwrap(); - let mut asm = Assembler::new(); + let mut asm = Assembler::new_without_iseq(); // gen_leave() fully reconstructs interpreter state and leaves the // return value in C_RET_OPND before coming here. let ret_opnd = asm.live_reg_opnd(C_RET_OPND); // Every exit to the interpreter should be counted - gen_counter_incr(&mut asm, Counter::leave_interp_return); + gen_counter_incr_without_pc(&mut asm, Counter::leave_interp_return); asm_comment!(asm, "exit from leave"); asm.cpop_into(SP); @@ -928,13 +1024,13 @@ fn gen_leave_exit(ocb: &mut OutlinedCb) -> Option<CodePtr> { // the caller's stack, which is different from gen_stub_exit(). fn gen_leave_exception(ocb: &mut OutlinedCb) -> Option<CodePtr> { let ocb = ocb.unwrap(); - let mut asm = Assembler::new(); + let mut asm = Assembler::new_without_iseq(); // gen_leave() leaves the return value in C_RET_OPND before coming here. let ruby_ret_val = asm.live_reg_opnd(C_RET_OPND); // Every exit to the interpreter should be counted - gen_counter_incr(&mut asm, Counter::leave_interp_return); + gen_counter_incr_without_pc(&mut asm, Counter::leave_interp_return); asm_comment!(asm, "push return value through cfp->sp"); let cfp_sp = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP); @@ -965,14 +1061,13 @@ fn gen_leave_exception(ocb: &mut OutlinedCb) -> Option<CodePtr> { pub fn gen_entry_chain_guard( asm: &mut Assembler, ocb: &mut OutlinedCb, - iseq: IseqPtr, - insn_idx: u16, + blockid: BlockId, ) -> Option<PendingEntryRef> { let entry = new_pending_entry(); let stub_addr = gen_entry_stub(entry.uninit_entry.as_ptr() as usize, ocb)?; let pc_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC); - let expected_pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx.into()) }; + let expected_pc = unsafe { rb_iseq_pc_at_idx(blockid.iseq, blockid.idx.into()) }; let expected_pc_opnd = Opnd::const_ptr(expected_pc as *const u8); asm_comment!(asm, "guard expected PC"); @@ -987,22 +1082,19 @@ pub fn gen_entry_chain_guard( /// Compile an interpreter entry block to be inserted into an iseq /// Returns None if compilation fails. /// If jit_exception is true, compile JIT code for handling exceptions. -/// See [jit_compile_exception] for details. +/// See jit_compile_exception() for details. pub fn gen_entry_prologue( cb: &mut CodeBlock, ocb: &mut OutlinedCb, - iseq: IseqPtr, - insn_idx: u16, + blockid: BlockId, + stack_size: u8, jit_exception: bool, -) -> Option<CodePtr> { +) -> Option<(CodePtr, RegMapping)> { + let iseq = blockid.iseq; let code_ptr = cb.get_write_ptr(); - let mut asm = Assembler::new(); - if get_option_ref!(dump_disasm).is_some() { - asm_comment!(asm, "YJIT entry point: {}", iseq_get_location(iseq, 0)); - } else { - asm_comment!(asm, "YJIT entry"); - } + let mut asm = Assembler::new(unsafe { get_iseq_body_local_table_size(iseq) }); + asm_comment!(asm, "YJIT entry point: {}", iseq_get_location(iseq, 0)); asm.frame_setup(); @@ -1049,10 +1141,11 @@ pub fn gen_entry_prologue( // If they don't match, then we'll jump to an entry stub and generate // another PC check and entry there. let pending_entry = if unsafe { get_iseq_flags_has_opt(iseq) } || jit_exception { - Some(gen_entry_chain_guard(&mut asm, ocb, iseq, insn_idx)?) + Some(gen_entry_chain_guard(&mut asm, ocb, blockid)?) } else { None }; + let reg_mapping = gen_entry_reg_mapping(&mut asm, blockid, stack_size); asm.compile(cb, Some(ocb))?; @@ -1070,10 +1163,39 @@ pub fn gen_entry_prologue( .ok().expect("PendingEntry should be unique"); iseq_payload.entries.push(pending_entry.into_entry()); } - Some(code_ptr) + Some((code_ptr, reg_mapping)) } } +/// Generate code to load registers for a JIT entry. When the entry block is compiled for +/// the first time, it loads no register. When it has been already compiled as a callee +/// block, it loads some registers to reuse the block. +pub fn gen_entry_reg_mapping(asm: &mut Assembler, blockid: BlockId, stack_size: u8) -> RegMapping { + // Find an existing callee block. If it's not found or uses no register, skip loading registers. + let mut ctx = Context::default(); + ctx.set_stack_size(stack_size); + let reg_mapping = find_most_compatible_reg_mapping(blockid, &ctx).unwrap_or(RegMapping::default()); + if reg_mapping == RegMapping::default() { + return reg_mapping; + } + + // If found, load the same registers to reuse the block. + asm_comment!(asm, "reuse maps: {:?}", reg_mapping); + let local_table_size: u32 = unsafe { get_iseq_body_local_table_size(blockid.iseq) }.try_into().unwrap(); + for ®_opnd in reg_mapping.get_reg_opnds().iter() { + match reg_opnd { + RegOpnd::Local(local_idx) => { + let loaded_reg = TEMP_REGS[reg_mapping.get_reg(reg_opnd).unwrap()]; + let loaded_temp = asm.local_opnd(local_table_size - local_idx as u32 + VM_ENV_DATA_SIZE - 1); + asm.load_into(Opnd::Reg(loaded_reg), loaded_temp); + } + RegOpnd::Stack(_) => unreachable!("find_most_compatible_reg_mapping should not leave {:?}", reg_opnd), + } + } + + reg_mapping +} + // Generate code to check for interrupts and take a side-exit. // Warning: this function clobbers REG0 fn gen_check_ints( @@ -1086,7 +1208,7 @@ fn gen_check_ints( // Not checking interrupt_mask since it's zero outside finalize_deferred_heap_pages, // signal_exec, or rb_postponed_job_flush. - let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG)); + let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32)); asm.test(interrupt_flag, interrupt_flag); asm.jnz(Target::side_exit(counter)); @@ -1097,8 +1219,15 @@ fn gen_check_ints( fn jump_to_next_insn( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, -) -> Option<()> { +) -> Option<CodegenStatus> { + end_block_with_jump(jit, asm, jit.next_insn_idx()) +} + +fn end_block_with_jump( + jit: &mut JITState, + asm: &mut Assembler, + continuation_insn_idx: u16, +) -> Option<CodegenStatus> { // Reset the depth since in current usages we only ever jump to // chain_depth > 0 from the same instruction. let mut reset_depth = asm.ctx; @@ -1106,20 +1235,20 @@ fn jump_to_next_insn( let jump_block = BlockId { iseq: jit.iseq, - idx: jit.next_insn_idx(), + idx: continuation_insn_idx, }; // We are at the end of the current instruction. Record the boundary. if jit.record_boundary_patch_point { jit.record_boundary_patch_point = false; - let exit_pc = unsafe { jit.pc.offset(insn_len(jit.opcode).try_into().unwrap()) }; - let exit_pos = gen_outlined_exit(exit_pc, &reset_depth, ocb); + let exit_pc = unsafe { rb_iseq_pc_at_idx(jit.iseq, continuation_insn_idx.into())}; + let exit_pos = jit.gen_outlined_exit(exit_pc, &reset_depth); record_global_inval_patch(asm, exit_pos?); } // Generate the jump instruction gen_direct_jump(jit, &reset_depth, jump_block, asm); - Some(()) + Some(EndBlock) } // Compile a sequence of bytecode instructions for a given basic block version. @@ -1132,6 +1261,7 @@ pub fn gen_single_block( ec: EcPtr, cb: &mut CodeBlock, ocb: &mut OutlinedCb, + first_block: bool, ) -> Result<BlockRef, ()> { // Limit the number of specialized versions for this block let ctx = limit_block_versions(blockid, start_ctx); @@ -1155,21 +1285,22 @@ pub fn gen_single_block( let mut insn_idx: IseqIdx = blockid.idx; // Initialize a JIT state object - let mut jit = JITState::new(blockid, ctx, cb.get_write_ptr(), ec); + let mut jit = JITState::new(blockid, ctx, cb.get_write_ptr(), ec, ocb, first_block); jit.iseq = blockid.iseq; // Create a backend assembler instance - let mut asm = Assembler::new(); + let mut asm = Assembler::new(jit.num_locals()); asm.ctx = ctx; - #[cfg(feature = "disasm")] if get_option_ref!(dump_disasm).is_some() { let blockid_idx = blockid.idx; let chain_depth = if asm.ctx.get_chain_depth() > 0 { format!("(chain_depth: {})", asm.ctx.get_chain_depth()) } else { "".to_string() }; asm_comment!(asm, "Block: {} {}", iseq_get_location(blockid.iseq, blockid_idx), chain_depth); - asm_comment!(asm, "reg_temps: {:08b}", asm.ctx.get_reg_temps().as_u8()); + asm_comment!(asm, "reg_mapping: {:?}", asm.ctx.get_reg_mapping()); } + Log::add_block_with_chain_depth(blockid, asm.ctx.get_chain_depth()); + // Mark the start of an ISEQ for --yjit-perf jit_perf_symbol_push!(jit, &mut asm, &get_iseq_name(iseq), PerfMap::ISEQ); @@ -1199,7 +1330,7 @@ pub fn gen_single_block( // if we run into it. This is necessary because we want to invalidate based on the // instruction's index. if opcode == YARVINSN_opt_getconstant_path.as_usize() && insn_idx > jit.starting_insn_idx { - jump_to_next_insn(&mut jit, &mut asm, ocb); + jump_to_next_insn(&mut jit, &mut asm); break; } @@ -1212,27 +1343,27 @@ pub fn gen_single_block( // stack_pop doesn't immediately deallocate a register for stack temps, // but it's safe to do so at this instruction boundary. - for stack_idx in asm.ctx.get_stack_size()..MAX_REG_TEMPS { - asm.ctx.dealloc_temp_reg(stack_idx); + for stack_idx in asm.ctx.get_stack_size()..MAX_CTX_TEMPS as u8 { + asm.ctx.dealloc_reg(RegOpnd::Stack(stack_idx)); } // If previous instruction requested to record the boundary if jit.record_boundary_patch_point { // Generate an exit to this instruction and record it - let exit_pos = gen_outlined_exit(jit.pc, &asm.ctx, ocb).ok_or(())?; + let exit_pos = jit.gen_outlined_exit(jit.pc, &asm.ctx).ok_or(())?; record_global_inval_patch(&mut asm, exit_pos); jit.record_boundary_patch_point = false; } // In debug mode, verify our existing assumption - if cfg!(debug_assertions) && get_option!(verify_ctx) && jit.at_current_insn() { + if cfg!(debug_assertions) && get_option!(verify_ctx) && jit.at_compile_target() { verify_ctx(&jit, &asm.ctx); } // :count-placement: // Count bytecode instructions that execute in generated code. // Note that the increment happens even when the output takes side exit. - gen_counter_incr(&mut asm, Counter::yjit_insns_count); + gen_counter_incr(&jit, &mut asm, Counter::yjit_insns_count); // Lookup the codegen function for this instruction let mut status = None; @@ -1248,7 +1379,7 @@ pub fn gen_single_block( // Call the code generation function jit_perf_symbol_push!(jit, &mut asm, &insn_name(opcode), PerfMap::Codegen); - status = gen_fn(&mut jit, &mut asm, ocb); + status = gen_fn(&mut jit, &mut asm); jit_perf_symbol_pop!(jit, &mut asm, PerfMap::Codegen); #[cfg(debug_assertions)] @@ -1283,13 +1414,6 @@ pub fn gen_single_block( // Move to the next instruction to compile insn_idx += insn_len(opcode) as u16; - // Move past next instruction when instructed - if status == Some(SkipNextInsn) { - let next_pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx.into()) }; - let next_opcode: usize = unsafe { rb_iseq_opcode_at_pc(iseq, next_pc) }.try_into().unwrap(); - insn_idx += insn_len(next_opcode) as u16; - } - // If the instruction terminates this block if status == Some(EndBlock) { break; @@ -1301,6 +1425,12 @@ pub fn gen_single_block( // doesn't go to the next instruction in the same iseq. assert!(!jit.record_boundary_patch_point); + // Bail when requested to. + if jit.block_abandoned { + incr_counter!(abandoned_block_count); + return Err(()); + } + // Pad the block if it has the potential to be invalidated if jit.block_entry_exit.is_some() { asm.pad_inval_patch(); @@ -1310,7 +1440,7 @@ pub fn gen_single_block( jit_perf_symbol_pop!(jit, &mut asm, PerfMap::ISEQ); // Compile code into the code block - let (_, gc_offsets) = asm.compile(cb, Some(ocb)).ok_or(())?; + let (_, gc_offsets) = asm.compile(cb, Some(jit.get_ocb())).ok_or(())?; let end_addr = cb.get_write_ptr(); // Flush perf symbols after asm.compile() writes addresses @@ -1319,7 +1449,7 @@ pub fn gen_single_block( } // If code for the block doesn't fit, fail - if cb.has_dropped_bytes() || ocb.unwrap().has_dropped_bytes() { + if cb.has_dropped_bytes() || jit.get_ocb().unwrap().has_dropped_bytes() { return Err(()); } @@ -1330,7 +1460,6 @@ pub fn gen_single_block( fn gen_nop( _jit: &mut JITState, _asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Do nothing Some(KeepCompiling) @@ -1339,7 +1468,6 @@ fn gen_nop( fn gen_pop( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Decrement SP asm.stack_pop(1); @@ -1349,7 +1477,6 @@ fn gen_pop( fn gen_dup( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let dup_val = asm.stack_opnd(0); let mapping = asm.ctx.get_opnd_mapping(dup_val.into()); @@ -1364,7 +1491,6 @@ fn gen_dup( fn gen_dupn( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let n = jit.get_arg(0).as_usize(); @@ -1388,11 +1514,22 @@ fn gen_dupn( Some(KeepCompiling) } +// Reverse top X stack entries +fn gen_opt_reverse( + jit: &mut JITState, + asm: &mut Assembler, +) -> Option<CodegenStatus> { + let count = jit.get_arg(0).as_i32(); + for n in 0..(count/2) { + stack_swap(asm, n, count - 1 - n); + } + Some(KeepCompiling) +} + // Swap top 2 stack entries fn gen_swap( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { stack_swap(asm, 0, 1); Some(KeepCompiling) @@ -1421,7 +1558,6 @@ fn stack_swap( fn gen_putnil( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { jit_putobject(asm, Qnil); Some(KeepCompiling) @@ -1436,7 +1572,6 @@ fn jit_putobject(asm: &mut Assembler, arg: VALUE) { fn gen_putobject_int2fix( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let opcode = jit.opcode; let cst_val: usize = if opcode == YARVINSN_putobject_INT2FIX_0_.as_usize() { @@ -1446,7 +1581,7 @@ fn gen_putobject_int2fix( }; let cst_val = VALUE::fixnum_from_usize(cst_val); - if let Some(result) = fuse_putobject_opt_ltlt(jit, asm, cst_val, ocb) { + if let Some(result) = fuse_putobject_opt_ltlt(jit, asm, cst_val) { return Some(result); } @@ -1457,11 +1592,10 @@ fn gen_putobject_int2fix( fn gen_putobject( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let arg: VALUE = jit.get_arg(0); - if let Some(result) = fuse_putobject_opt_ltlt(jit, asm, arg, ocb) { + if let Some(result) = fuse_putobject_opt_ltlt(jit, asm, arg) { return Some(result); } @@ -1475,7 +1609,6 @@ fn fuse_putobject_opt_ltlt( jit: &mut JITState, asm: &mut Assembler, constant_object: VALUE, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let next_opcode = unsafe { rb_vm_insn_addr2opcode(jit.pc.add(insn_len(jit.opcode).as_usize()).read().as_ptr()) }; if next_opcode == YARVINSN_opt_ltlt as i32 && constant_object.fixnum_p() { @@ -1484,9 +1617,8 @@ fn fuse_putobject_opt_ltlt( if shift_amt > 63 || shift_amt < 0 { return None; } - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } let lhs = jit.peek_at_stack(&asm.ctx, 0); @@ -1494,7 +1626,7 @@ fn fuse_putobject_opt_ltlt( return None; } - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_LTLT) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, BOP_LTLT) { return None; } @@ -1511,7 +1643,6 @@ fn fuse_putobject_opt_ltlt( JCC_JZ, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::guard_send_not_fixnums, ); @@ -1519,7 +1650,7 @@ fn fuse_putobject_opt_ltlt( asm.stack_pop(1); fixnum_left_shift_body(asm, lhs, shift_amt as u64); - return Some(SkipNextInsn); + return end_block_with_jump(jit, asm, jit.next_next_insn_idx()); } return None; } @@ -1527,7 +1658,6 @@ fn fuse_putobject_opt_ltlt( fn gen_putself( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Write it on the stack @@ -1543,7 +1673,6 @@ fn gen_putself( fn gen_putspecialobject( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let object_type = jit.get_arg(0).as_usize(); @@ -1563,7 +1692,6 @@ fn gen_putspecialobject( fn gen_setn( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let n = jit.get_arg(0).as_usize(); @@ -1584,7 +1712,6 @@ fn gen_setn( fn gen_topn( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let n = jit.get_arg(0).as_usize(); @@ -1600,7 +1727,6 @@ fn gen_topn( fn gen_adjuststack( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let n = jit.get_arg(0).as_usize(); asm.stack_pop(n); @@ -1610,23 +1736,21 @@ fn gen_adjuststack( fn gen_opt_plus( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let two_fixnums = match asm.ctx.two_fixnums_on_stack(jit) { Some(two_fixnums) => two_fixnums, None => { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + return jit.defer_compilation(asm); } }; if two_fixnums { - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) { return None; } // Check that both operands are fixnums - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); // Get the operands from the stack let arg1 = asm.stack_pop(1); @@ -1643,7 +1767,7 @@ fn gen_opt_plus( Some(KeepCompiling) } else { - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } @@ -1651,7 +1775,6 @@ fn gen_opt_plus( fn gen_newarray( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let n = jit.get_arg(0).as_u32(); @@ -1688,7 +1811,6 @@ fn gen_newarray( fn gen_duparray( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let ary = jit.get_arg(0); @@ -1711,7 +1833,6 @@ fn gen_duparray( fn gen_duphash( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let hash = jit.get_arg(0); @@ -1731,7 +1852,6 @@ fn gen_duphash( fn gen_splatarray( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let flag = jit.get_arg(0).as_usize(); @@ -1757,12 +1877,10 @@ fn gen_splatarray( fn gen_splatkw( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Defer compilation so we can specialize on a runtime hash operand - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } let comptime_hash = jit.peek_at_stack(&asm.ctx, 1); @@ -1810,7 +1928,7 @@ fn gen_splatkw( asm.mov(stack_ret, hash); asm.stack_push(block_type); // Leave block_opnd spilled by ccall as is - asm.ctx.dealloc_temp_reg(asm.ctx.get_stack_size() - 1); + asm.ctx.dealloc_reg(RegOpnd::Stack(asm.ctx.get_stack_size() - 1)); } Some(KeepCompiling) @@ -1820,7 +1938,6 @@ fn gen_splatkw( fn gen_concatarray( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Save the PC and SP because the callee may call #to_a // Note that this modifies REG_SP, which is why we do it first @@ -1846,7 +1963,6 @@ fn gen_concatarray( fn gen_concattoarray( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Save the PC and SP because the callee may call #to_a jit_prepare_non_leaf_call(jit, asm); @@ -1868,7 +1984,6 @@ fn gen_concattoarray( fn gen_pushtoarray( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let num = jit.get_arg(0).as_u64(); @@ -1892,7 +2007,6 @@ fn gen_pushtoarray( fn gen_newrange( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let flag = jit.get_arg(0).as_usize(); @@ -2006,6 +2120,46 @@ fn guard_object_is_hash( } } +fn guard_object_is_fixnum( + jit: &mut JITState, + asm: &mut Assembler, + object: Opnd, + object_opnd: YARVOpnd +) { + let object_type = asm.ctx.get_opnd_type(object_opnd); + if object_type.is_heap() { + asm_comment!(asm, "arg is heap object"); + asm.jmp(Target::side_exit(Counter::guard_send_not_fixnum)); + return; + } + + if object_type != Type::Fixnum && object_type.is_specific() { + asm_comment!(asm, "arg is not fixnum"); + asm.jmp(Target::side_exit(Counter::guard_send_not_fixnum)); + return; + } + + assert!(!object_type.is_heap()); + assert!(object_type == Type::Fixnum || object_type.is_unknown()); + + // If not fixnums at run-time, fall back + if object_type != Type::Fixnum { + asm_comment!(asm, "guard object fixnum"); + asm.test(object, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); + + jit_chain_guard( + JCC_JZ, + jit, + asm, + SEND_MAX_DEPTH, + Counter::guard_send_not_fixnum, + ); + } + + // Set the stack type in the context. + asm.ctx.upgrade_opnd_type(object.into(), Type::Fixnum); +} + fn guard_object_is_string( asm: &mut Assembler, object: Opnd, @@ -2078,7 +2232,6 @@ fn guard_object_is_not_ruby2_keyword_hash( fn gen_expandarray( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Both arguments are rb_num_t which is unsigned let num = jit.get_arg(0).as_u32(); @@ -2086,27 +2239,27 @@ fn gen_expandarray( // If this instruction has the splat flag, then bail out. if flag & 0x01 != 0 { - gen_counter_incr(asm, Counter::expandarray_splat); + gen_counter_incr(jit, asm, Counter::expandarray_splat); return None; } // If this instruction has the postarg flag, then bail out. if flag & 0x02 != 0 { - gen_counter_incr(asm, Counter::expandarray_postarg); + gen_counter_incr(jit, asm, Counter::expandarray_postarg); return None; } let array_opnd = asm.stack_opnd(0); // Defer compilation so we can specialize on a runtime `self` - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } let comptime_recv = jit.peek_at_stack(&asm.ctx, 0); - // If the comptime receiver is not an array + // If the comptime receiver is not an array, speculate for when the `rb_check_array_type()` + // conversion returns nil and without side-effects (e.g. arbitrary method calls). if !unsafe { RB_TYPE_P(comptime_recv, RUBY_T_ARRAY) } { // at compile time, ensure to_ary is not defined let target_cme = unsafe { rb_callable_method_entry_or_negative(comptime_recv.class_of(), ID!(to_ary)) }; @@ -2114,18 +2267,23 @@ fn gen_expandarray( // if to_ary is defined, return can't compile so to_ary can be called if cme_def_type != VM_METHOD_TYPE_UNDEF { - gen_counter_incr(asm, Counter::expandarray_to_ary); + gen_counter_incr(jit, asm, Counter::expandarray_to_ary); + return None; + } + + // Bail when method_missing is defined to avoid generating code to call it. + // Also, for simplicity, bail when BasicObject#method_missing has been removed. + if !assume_method_basic_definition(jit, asm, comptime_recv.class_of(), ID!(method_missing)) { + gen_counter_incr(jit, asm, Counter::expandarray_method_missing); return None; } // invalidate compile block if to_ary is later defined - jit.assume_method_lookup_stable(asm, ocb, target_cme); + jit.assume_method_lookup_stable(asm, target_cme); jit_guard_known_klass( jit, asm, - ocb, - comptime_recv.class_of(), array_opnd, array_opnd.into(), comptime_recv, @@ -2155,7 +2313,7 @@ fn gen_expandarray( } // Get the compile-time array length - let comptime_len = unsafe { rb_yjit_array_len(comptime_recv) as u32 }; + let comptime_len = unsafe { rb_jit_array_len(comptime_recv) as u32 }; // Move the array from the stack and check that it's an array. guard_object_is_array( @@ -2183,7 +2341,6 @@ fn gen_expandarray( JCC_JB, jit, asm, - ocb, EXPANDARRAY_MAX_CHAIN_DEPTH, Counter::expandarray_chain_max_depth, ); @@ -2195,7 +2352,6 @@ fn gen_expandarray( JCC_JNE, jit, asm, - ocb, EXPANDARRAY_MAX_CHAIN_DEPTH, Counter::expandarray_chain_max_depth, ); @@ -2288,13 +2444,17 @@ fn gen_get_lep(jit: &JITState, asm: &mut Assembler) -> Opnd { fn gen_getlocal_generic( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ep_offset: u32, level: u32, ) -> Option<CodegenStatus> { - let local_opnd = if level == 0 && jit.assume_no_ep_escape(asm, ocb) { + // Split the block if we need to invalidate this instruction when EP escapes + if level == 0 && !jit.escapes_ep() && !jit.at_compile_target() { + return jit.defer_compilation(asm); + } + + let local_opnd = if level == 0 && jit.assume_no_ep_escape(asm) { // Load the local using SP register - asm.ctx.ep_opnd(-(ep_offset as i32)) + asm.local_opnd(ep_offset) } else { // Load environment pointer EP (level 0) from CFP let ep_opnd = gen_get_ep(asm, level); @@ -2302,7 +2462,16 @@ fn gen_getlocal_generic( // Load the local from the block // val = *(vm_get_ep(GET_EP(), level) - idx); let offs = -(SIZEOF_VALUE_I32 * ep_offset as i32); - Opnd::mem(64, ep_opnd, offs) + let local_opnd = Opnd::mem(64, ep_opnd, offs); + + // Write back an argument register to the stack. If the local variable + // is an argument, it might have an allocated register, but if this ISEQ + // is known to escape EP, the register shouldn't be used after this getlocal. + if level == 0 && asm.ctx.get_reg_mapping().get_reg(asm.local_opnd(ep_offset).reg_opnd()).is_some() { + asm.mov(local_opnd, asm.local_opnd(ep_offset)); + } + + local_opnd }; // Write the local at SP @@ -2321,38 +2490,35 @@ fn gen_getlocal_generic( fn gen_getlocal( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let idx = jit.get_arg(0).as_u32(); let level = jit.get_arg(1).as_u32(); - gen_getlocal_generic(jit, asm, ocb, idx, level) + gen_getlocal_generic(jit, asm, idx, level) } fn gen_getlocal_wc0( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let idx = jit.get_arg(0).as_u32(); - gen_getlocal_generic(jit, asm, ocb, idx, 0) + gen_getlocal_generic(jit, asm, idx, 0) } fn gen_getlocal_wc1( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let idx = jit.get_arg(0).as_u32(); - gen_getlocal_generic(jit, asm, ocb, idx, 1) + gen_getlocal_generic(jit, asm, idx, 1) } fn gen_setlocal_generic( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ep_offset: u32, level: u32, ) -> Option<CodegenStatus> { + // Post condition: The type of of the set local is updated in the Context. let value_type = asm.ctx.get_opnd_type(StackOpnd(0)); // Fallback because of write barrier @@ -2374,15 +2540,35 @@ fn gen_setlocal_generic( ); asm.stack_pop(1); + // Set local type in the context + if level == 0 { + let local_idx = ep_offset_to_local_idx(jit.get_iseq(), ep_offset).as_usize(); + asm.ctx.set_local_type(local_idx, value_type); + } return Some(KeepCompiling); } - let (flags_opnd, local_opnd) = if level == 0 && jit.assume_no_ep_escape(asm, ocb) { + // Split the block if we need to invalidate this instruction when EP escapes + if level == 0 && !jit.escapes_ep() && !jit.at_compile_target() { + return jit.defer_compilation(asm); + } + + let (flags_opnd, local_opnd) = if level == 0 && jit.assume_no_ep_escape(asm) { // Load flags and the local using SP register - let local_opnd = asm.ctx.ep_opnd(-(ep_offset as i32)); let flags_opnd = asm.ctx.ep_opnd(VM_ENV_DATA_INDEX_FLAGS as i32); + let local_opnd = asm.local_opnd(ep_offset); + + // Allocate a register to the new local operand + asm.alloc_reg(local_opnd.reg_opnd()); (flags_opnd, local_opnd) } else { + // Make sure getlocal doesn't read a stale register. If the local variable + // is an argument, it might have an allocated register, but if this ISEQ + // is known to escape EP, the register shouldn't be used after this setlocal. + if level == 0 { + asm.ctx.dealloc_reg(asm.local_opnd(ep_offset).reg_opnd()); + } + // Load flags and the local for the level let ep_opnd = gen_get_ep(asm, level); let flags_opnd = Opnd::mem( @@ -2406,12 +2592,12 @@ fn gen_setlocal_generic( JCC_JNZ, jit, asm, - ocb, 1, Counter::setlocal_wb_required, ); } + // Set local type in the context if level == 0 { let local_idx = ep_offset_to_local_idx(jit.get_iseq(), ep_offset).as_usize(); asm.ctx.set_local_type(local_idx, value_type); @@ -2429,36 +2615,32 @@ fn gen_setlocal_generic( fn gen_setlocal( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let idx = jit.get_arg(0).as_u32(); let level = jit.get_arg(1).as_u32(); - gen_setlocal_generic(jit, asm, ocb, idx, level) + gen_setlocal_generic(jit, asm, idx, level) } fn gen_setlocal_wc0( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let idx = jit.get_arg(0).as_u32(); - gen_setlocal_generic(jit, asm, ocb, idx, 0) + gen_setlocal_generic(jit, asm, idx, 0) } fn gen_setlocal_wc1( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let idx = jit.get_arg(0).as_u32(); - gen_setlocal_generic(jit, asm, ocb, idx, 1) + gen_setlocal_generic(jit, asm, idx, 1) } // new hash initialized from top N values fn gen_newhash( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let num: u64 = jit.get_arg(0).as_u64(); @@ -2508,7 +2690,6 @@ fn gen_newhash( fn gen_putstring( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let put_val = jit.get_arg(0); @@ -2529,7 +2710,6 @@ fn gen_putstring( fn gen_putchilledstring( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let put_val = jit.get_arg(0); @@ -2550,7 +2730,6 @@ fn gen_putchilledstring( fn gen_checkmatch( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let flag = jit.get_arg(0).as_u32(); @@ -2580,11 +2759,10 @@ fn gen_checkmatch( fn gen_checkkeyword( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // When a keyword is unspecified past index 32, a hash will be used // instead. This can only happen in iseqs taking more than 32 keywords. - if unsafe { (*get_iseq_body_param_keyword(jit.iseq)).num >= 32 } { + if unsafe { (*get_iseq_body_param_keyword(jit.iseq)).num >= VM_KW_SPECIFIED_BITS_MAX.try_into().unwrap() } { return None; } @@ -2594,11 +2772,11 @@ fn gen_checkkeyword( // The index of the keyword we want to check let index: i64 = jit.get_arg(1).as_i64(); - // Load environment pointer EP - let ep_opnd = gen_get_ep(asm, 0); - - // VALUE kw_bits = *(ep - bits); - let bits_opnd = Opnd::mem(64, ep_opnd, SIZEOF_VALUE_I32 * -bits_offset); + // `unspecified_bits` is a part of the local table. Therefore, we may allocate a register for + // that "local" when passing it as an argument. We must use such a register to avoid loading + // random bits from the stack if any. We assume that EP is not escaped as of entering a method + // with keyword arguments. + let bits_opnd = asm.local_opnd(bits_offset as u32); // unsigned int b = (unsigned int)FIX2ULONG(kw_bits); // if ((b & (0x01 << idx))) { @@ -2620,7 +2798,6 @@ fn jit_chain_guard( jcc: JCCKinds, jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, depth_limit: u8, counter: Counter, ) { @@ -2641,7 +2818,7 @@ fn jit_chain_guard( idx: jit.insn_idx, }; - gen_branch(jit, asm, ocb, bid, &deeper, None, None, target0_gen_fn); + jit.gen_branch(asm, bid, &deeper, None, None, target0_gen_fn); } else { target0_gen_fn.call(asm, Target::side_exit(counter), None); } @@ -2674,45 +2851,29 @@ pub const MAX_SPLAT_LENGTH: i32 = 127; fn gen_get_ivar( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, max_chain_depth: u8, comptime_receiver: VALUE, ivar_name: ID, recv: Opnd, recv_opnd: YARVOpnd, ) -> Option<CodegenStatus> { - let comptime_val_klass = comptime_receiver.class_of(); - // If recv isn't already a register, load it. let recv = match recv { Opnd::InsnOut { .. } => recv, _ => asm.load(recv), }; - // Check if the comptime class uses a custom allocator - let custom_allocator = unsafe { rb_get_alloc_func(comptime_val_klass) }; - let uses_custom_allocator = match custom_allocator { - Some(alloc_fun) => { - let allocate_instance = rb_class_allocate_instance as *const u8; - alloc_fun as *const u8 != allocate_instance - } - None => false, - }; - // Check if the comptime receiver is a T_OBJECT let receiver_t_object = unsafe { RB_TYPE_P(comptime_receiver, RUBY_T_OBJECT) }; // Use a general C call at the last chain to avoid exits on megamorphic shapes let megamorphic = asm.ctx.get_chain_depth() >= max_chain_depth; if megamorphic { - gen_counter_incr(asm, Counter::num_getivar_megamorphic); + gen_counter_incr(jit, asm, Counter::num_getivar_megamorphic); } - // If the class uses the default allocator, instances should all be T_OBJECT - // NOTE: This assumes nobody changes the allocator of the class after allocation. - // Eventually, we can encode whether an object is T_OBJECT or not - // inside object shapes. + // NOTE: This assumes T_OBJECT can't ever have the same shape_id as any other type. // too-complex shapes can't use index access, so we use rb_ivar_get for them too. - if !receiver_t_object || uses_custom_allocator || comptime_receiver.shape_too_complex() || megamorphic { + if !comptime_receiver.heap_object_p() || comptime_receiver.shape_too_complex() || megamorphic { // General case. Call rb_ivar_get(). // VALUE rb_ivar_get(VALUE obj, ID id) asm_comment!(asm, "call rb_ivar_get()"); @@ -2731,15 +2892,14 @@ fn gen_get_ivar( asm.mov(out_opnd, ivar_val); // Jump to next instruction. This allows guard chains to share the same successor. - jump_to_next_insn(jit, asm, ocb); + jump_to_next_insn(jit, asm); return Some(EndBlock); } let ivar_index = unsafe { let shape_id = comptime_receiver.shape_id_of(); - let shape = rb_shape_get_shape_by_id(shape_id); - let mut ivar_index: u32 = 0; - if rb_shape_get_iv_index(shape, ivar_name, &mut ivar_index) { + let mut ivar_index: u16 = 0; + if rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) { Some(ivar_index as usize) } else { None @@ -2749,10 +2909,7 @@ fn gen_get_ivar( // Guard heap object (recv_opnd must be used before stack_pop) guard_object_is_heap(asm, recv, recv_opnd, Counter::getivar_not_heap); - // Compile time self is embedded and the ivar index lands within the object - let embed_test_result = unsafe { FL_TEST_RAW(comptime_receiver, VALUE(ROBJECT_EMBED.as_usize())) != VALUE(0) }; - - let expected_shape = unsafe { rb_shape_get_shape_id(comptime_receiver) }; + let expected_shape = unsafe { rb_obj_shape_id(comptime_receiver) }; let shape_id_offset = unsafe { rb_shape_id_offset() }; let shape_opnd = Opnd::mem(SHAPE_ID_NUM_BITS as u8, recv, shape_id_offset); @@ -2762,7 +2919,6 @@ fn gen_get_ivar( JCC_JNE, jit, asm, - ocb, max_chain_depth, Counter::getivar_megamorphic, ); @@ -2781,45 +2937,52 @@ fn gen_get_ivar( asm.mov(out_opnd, Qnil.into()); } Some(ivar_index) => { - if embed_test_result { - // See ROBJECT_IVPTR() from include/ruby/internal/core/robject.h - - // Load the variable - let offs = ROBJECT_OFFSET_AS_ARY as i32 + (ivar_index * SIZEOF_VALUE) as i32; - let ivar_opnd = Opnd::mem(64, recv, offs); - - // Push the ivar on the stack - let out_opnd = asm.stack_push(Type::Unknown); - asm.mov(out_opnd, ivar_opnd); + let ivar_opnd = if receiver_t_object { + if comptime_receiver.embedded_p() { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + + // Load the variable + let offs = ROBJECT_OFFSET_AS_ARY as i32 + (ivar_index * SIZEOF_VALUE) as i32; + Opnd::mem(64, recv, offs) + } else { + // Compile time value is *not* embedded. + + // Get a pointer to the extended table + let tbl_opnd = asm.load(Opnd::mem(64, recv, ROBJECT_OFFSET_AS_HEAP_FIELDS as i32)); + + // Read the ivar from the extended table + Opnd::mem(64, tbl_opnd, (SIZEOF_VALUE * ivar_index) as i32) + } } else { - // Compile time value is *not* embedded. - - // Get a pointer to the extended table - let tbl_opnd = asm.load(Opnd::mem(64, recv, ROBJECT_OFFSET_AS_HEAP_IVPTR as i32)); + asm_comment!(asm, "call rb_ivar_get_at()"); - // Read the ivar from the extended table - let ivar_opnd = Opnd::mem(64, tbl_opnd, (SIZEOF_VALUE * ivar_index) as i32); + if assume_single_ractor_mode(jit, asm) { + asm.ccall(rb_ivar_get_at_no_ractor_check as *const u8, vec![recv, Opnd::UImm((ivar_index as u32).into())]) + } else { + // The function could raise RactorIsolationError. + jit_prepare_non_leaf_call(jit, asm); + asm.ccall(rb_ivar_get_at as *const u8, vec![recv, Opnd::UImm((ivar_index as u32).into()), Opnd::UImm(ivar_name)]) + } + }; - let out_opnd = asm.stack_push(Type::Unknown); - asm.mov(out_opnd, ivar_opnd); - } + // Push the ivar on the stack + let out_opnd = asm.stack_push(Type::Unknown); + asm.mov(out_opnd, ivar_opnd); } } // Jump to next instruction. This allows guard chains to share the same successor. - jump_to_next_insn(jit, asm, ocb); + jump_to_next_insn(jit, asm); Some(EndBlock) } fn gen_getinstancevariable( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Defer compilation so we can specialize on a runtime `self` - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } let ivar_name = jit.get_arg(0).as_u64(); @@ -2832,7 +2995,6 @@ fn gen_getinstancevariable( gen_get_ivar( jit, asm, - ocb, GET_IVAR_MAX_DEPTH, comptime_val, ivar_name, @@ -2867,7 +3029,7 @@ fn gen_write_iv( // Compile time value is *not* embedded. // Get a pointer to the extended table - let tbl_opnd = asm.load(Opnd::mem(64, recv, ROBJECT_OFFSET_AS_HEAP_IVPTR as i32)); + let tbl_opnd = asm.load(Opnd::mem(64, recv, ROBJECT_OFFSET_AS_HEAP_FIELDS as i32)); // Write the ivar in to the extended table let ivar_opnd = Opnd::mem(64, tbl_opnd, (SIZEOF_VALUE * ivar_index) as i32); @@ -2880,12 +3042,10 @@ fn gen_write_iv( fn gen_setinstancevariable( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Defer compilation so we can specialize on a runtime `self` - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } let ivar_name = jit.get_arg(0).as_u64(); @@ -2894,7 +3054,6 @@ fn gen_setinstancevariable( gen_set_ivar( jit, asm, - ocb, comptime_receiver, ivar_name, SelfOpnd, @@ -2909,48 +3068,34 @@ fn gen_setinstancevariable( fn gen_set_ivar( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, comptime_receiver: VALUE, ivar_name: ID, recv_opnd: YARVOpnd, ic: Option<*const iseq_inline_iv_cache_entry>, ) -> Option<CodegenStatus> { - let comptime_val_klass = comptime_receiver.class_of(); - // If the comptime receiver is frozen, writing an IV will raise an exception // and we don't want to JIT code to deal with that situation. if comptime_receiver.is_frozen() { - gen_counter_incr(asm, Counter::setivar_frozen); + gen_counter_incr(jit, asm, Counter::setivar_frozen); return None; } let stack_type = asm.ctx.get_opnd_type(StackOpnd(0)); - // Check if the comptime class uses a custom allocator - let custom_allocator = unsafe { rb_get_alloc_func(comptime_val_klass) }; - let uses_custom_allocator = match custom_allocator { - Some(alloc_fun) => { - let allocate_instance = rb_class_allocate_instance as *const u8; - alloc_fun as *const u8 != allocate_instance - } - None => false, - }; - // Check if the comptime receiver is a T_OBJECT let receiver_t_object = unsafe { RB_TYPE_P(comptime_receiver, RUBY_T_OBJECT) }; // Use a general C call at the last chain to avoid exits on megamorphic shapes let megamorphic = asm.ctx.get_chain_depth() >= SET_IVAR_MAX_DEPTH; if megamorphic { - gen_counter_incr(asm, Counter::num_setivar_megamorphic); + gen_counter_incr(jit, asm, Counter::num_setivar_megamorphic); } // Get the iv index let shape_too_complex = comptime_receiver.shape_too_complex(); - let ivar_index = if !shape_too_complex { + let ivar_index = if !comptime_receiver.special_const_p() && !shape_too_complex { let shape_id = comptime_receiver.shape_id_of(); - let shape = unsafe { rb_shape_get_shape_by_id(shape_id) }; - let mut ivar_index: u32 = 0; - if unsafe { rb_shape_get_iv_index(shape, ivar_name, &mut ivar_index) } { + let mut ivar_index: u16 = 0; + if unsafe { rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) } { Some(ivar_index as usize) } else { None @@ -2960,27 +3105,31 @@ fn gen_set_ivar( }; // The current shape doesn't contain this iv, we need to transition to another shape. + let mut new_shape_too_complex = false; let new_shape = if !shape_too_complex && receiver_t_object && ivar_index.is_none() { - let current_shape = comptime_receiver.shape_of(); - let next_shape = unsafe { rb_shape_get_next(current_shape, comptime_receiver, ivar_name) }; - let next_shape_id = unsafe { rb_shape_id(next_shape) }; + let current_shape_id = comptime_receiver.shape_id_of(); + // We don't need to check about imemo_fields here because we're definitely looking at a T_OBJECT. + let klass = unsafe { rb_obj_class(comptime_receiver) }; + let next_shape_id = unsafe { rb_shape_transition_add_ivar_no_warnings(klass, current_shape_id, ivar_name) }; // If the VM ran out of shapes, or this class generated too many leaf, // it may be de-optimized into OBJ_TOO_COMPLEX_SHAPE (hash-table). - if next_shape_id == OBJ_TOO_COMPLEX_SHAPE_ID { + new_shape_too_complex = unsafe { rb_jit_shape_too_complex_p(next_shape_id) }; + if new_shape_too_complex { Some((next_shape_id, None, 0_usize)) } else { - let current_capacity = unsafe { (*current_shape).capacity }; + let current_capacity = unsafe { rb_yjit_shape_capacity(current_shape_id) }; + let next_capacity = unsafe { rb_yjit_shape_capacity(next_shape_id) }; // If the new shape has a different capacity, or is TOO_COMPLEX, we'll have to // reallocate it. - let needs_extension = unsafe { (*current_shape).capacity != (*next_shape).capacity }; + let needs_extension = next_capacity != current_capacity; // We can write to the object, but we need to transition the shape - let ivar_index = unsafe { (*current_shape).next_iv_index } as usize; + let ivar_index = unsafe { rb_yjit_shape_index(next_shape_id) } as usize; let needs_extension = if needs_extension { - Some((current_capacity, unsafe { (*next_shape).capacity })) + Some((current_capacity, next_capacity)) } else { None }; @@ -2989,12 +3138,10 @@ fn gen_set_ivar( } else { None }; - let new_shape_too_complex = matches!(new_shape, Some((OBJ_TOO_COMPLEX_SHAPE_ID, _, _))); - // If the receiver isn't a T_OBJECT, or uses a custom allocator, - // then just write out the IV write as a function call. + // If the receiver isn't a T_OBJECT, then just write out the IV write as a function call. // too-complex shapes can't use index access, so we use rb_ivar_get for them too. - if !receiver_t_object || uses_custom_allocator || shape_too_complex || new_shape_too_complex || megamorphic { + if !receiver_t_object || shape_too_complex || new_shape_too_complex || megamorphic { // The function could raise FrozenError. // Note that this modifies REG_SP, which is why we do it first jit_prepare_non_leaf_call(jit, asm); @@ -3018,7 +3165,7 @@ fn gen_set_ivar( asm.ccall( rb_vm_setinstancevariable as *const u8, vec![ - Opnd::const_ptr(jit.iseq as *const u8), + VALUE(jit.iseq as usize).into(), Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF), ivar_name.into(), val_opnd, @@ -3037,7 +3184,7 @@ fn gen_set_ivar( // Upgrade type guard_object_is_heap(asm, recv, recv_opnd, Counter::setivar_not_heap); - let expected_shape = unsafe { rb_shape_get_shape_id(comptime_receiver) }; + let expected_shape = unsafe { rb_obj_shape_id(comptime_receiver) }; let shape_id_offset = unsafe { rb_shape_id_offset() }; let shape_opnd = Opnd::mem(SHAPE_ID_NUM_BITS as u8, recv, shape_id_offset); @@ -3047,7 +3194,6 @@ fn gen_set_ivar( JCC_JNE, jit, asm, - ocb, SET_IVAR_MAX_DEPTH, Counter::setivar_megamorphic, ); @@ -3109,7 +3255,7 @@ fn gen_set_ivar( // If we know the stack value is an immediate, there's no need to // generate WB code. if !stack_type.is_imm() { - asm.spill_temps(); // for ccall (unconditionally spill them for RegTemps consistency) + asm.spill_regs(); // for ccall (unconditionally spill them for RegMappings consistency) let skip_wb = asm.new_label("skip_wb"); // If the value we're writing is an immediate, we don't need to WB asm.test(write_val, (RUBY_IMMEDIATE_MASK as u64).into()); @@ -3148,7 +3294,6 @@ fn gen_set_ivar( fn gen_defined( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let op_type = jit.get_arg(0).as_u64(); let obj = jit.get_arg(1); @@ -3196,12 +3341,10 @@ fn gen_defined( fn gen_definedivar( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Defer compilation so we can specialize base on a runtime receiver - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } let ivar_name = jit.get_arg(0).as_u64(); @@ -3215,7 +3358,7 @@ fn gen_definedivar( // Specialize base on compile time values let comptime_receiver = jit.peek_at_self(); - if comptime_receiver.shape_too_complex() || asm.ctx.get_chain_depth() >= GET_IVAR_MAX_DEPTH { + if comptime_receiver.special_const_p() || comptime_receiver.shape_too_complex() || asm.ctx.get_chain_depth() >= GET_IVAR_MAX_DEPTH { // Fall back to calling rb_ivar_defined // Save the PC and SP because the callee may allocate @@ -3241,9 +3384,8 @@ fn gen_definedivar( let shape_id = comptime_receiver.shape_id_of(); let ivar_exists = unsafe { - let shape = rb_shape_get_shape_by_id(shape_id); - let mut ivar_index: u32 = 0; - rb_shape_get_iv_index(shape, ivar_name, &mut ivar_index) + let mut ivar_index: u16 = 0; + rb_shape_get_iv_index(shape_id, ivar_name, &mut ivar_index) }; // Guard heap object (recv_opnd must be used before stack_pop) @@ -3258,7 +3400,6 @@ fn gen_definedivar( JCC_JNE, jit, asm, - ocb, GET_IVAR_MAX_DEPTH, Counter::definedivar_megamorphic, ); @@ -3267,15 +3408,12 @@ fn gen_definedivar( jit_putobject(asm, result); // Jump to next instruction. This allows guard chains to share the same successor. - jump_to_next_insn(jit, asm, ocb); - - return Some(EndBlock); + return jump_to_next_insn(jit, asm); } fn gen_checktype( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let type_val = jit.get_arg(0).as_u32(); @@ -3330,7 +3468,6 @@ fn gen_checktype( fn gen_concatstrings( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let n = jit.get_arg(0).as_usize(); @@ -3355,7 +3492,6 @@ fn gen_concatstrings( fn guard_two_fixnums( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) { let counter = Counter::guard_send_not_fixnums; @@ -3399,7 +3535,6 @@ fn guard_two_fixnums( JCC_JZ, jit, asm, - ocb, SEND_MAX_DEPTH, counter, ); @@ -3412,7 +3547,6 @@ fn guard_two_fixnums( JCC_JZ, jit, asm, - ocb, SEND_MAX_DEPTH, counter, ); @@ -3429,7 +3563,6 @@ type CmovFn = fn(cb: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> Opnd; fn gen_fixnum_cmp( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, cmov_op: CmovFn, bop: ruby_basic_operators, ) -> Option<CodegenStatus> { @@ -3437,18 +3570,17 @@ fn gen_fixnum_cmp( Some(two_fixnums) => two_fixnums, None => { // Defer compilation so we can specialize based on a runtime receiver - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + return jit.defer_compilation(asm); } }; if two_fixnums { - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, bop) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, bop) { return None; } // Check that both operands are fixnums - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); // Get the operands from the stack let arg1 = asm.stack_pop(1); @@ -3464,40 +3596,36 @@ fn gen_fixnum_cmp( Some(KeepCompiling) } else { - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } fn gen_opt_lt( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - gen_fixnum_cmp(jit, asm, ocb, Assembler::csel_l, BOP_LT) + gen_fixnum_cmp(jit, asm, Assembler::csel_l, BOP_LT) } fn gen_opt_le( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - gen_fixnum_cmp(jit, asm, ocb, Assembler::csel_le, BOP_LE) + gen_fixnum_cmp(jit, asm, Assembler::csel_le, BOP_LE) } fn gen_opt_ge( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - gen_fixnum_cmp(jit, asm, ocb, Assembler::csel_ge, BOP_GE) + gen_fixnum_cmp(jit, asm, Assembler::csel_ge, BOP_GE) } fn gen_opt_gt( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - gen_fixnum_cmp(jit, asm, ocb, Assembler::csel_g, BOP_GT) + gen_fixnum_cmp(jit, asm, Assembler::csel_g, BOP_GT) } // Implements specialized equality for either two fixnum or two strings @@ -3506,7 +3634,6 @@ fn gen_opt_gt( fn gen_equality_specialized( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, gen_eq: bool, ) -> Option<bool> { let a_opnd = asm.stack_opnd(1); @@ -3518,12 +3645,12 @@ fn gen_equality_specialized( }; if two_fixnums { - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_EQ) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, BOP_EQ) { // if overridden, emit the generic version return Some(false); } - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); asm.cmp(a_opnd, b_opnd); let val = if gen_eq { @@ -3540,14 +3667,14 @@ fn gen_equality_specialized( return Some(true); } - if !jit.at_current_insn() { + if !jit.at_compile_target() { return None; } let comptime_a = jit.peek_at_stack(&asm.ctx, 1); let comptime_b = jit.peek_at_stack(&asm.ctx, 0); if unsafe { comptime_a.class_of() == rb_cString && comptime_b.class_of() == rb_cString } { - if !assume_bop_not_redefined(jit, asm, ocb, STRING_REDEFINED_OP_FLAG, BOP_EQ) { + if !assume_bop_not_redefined(jit, asm, STRING_REDEFINED_OP_FLAG, BOP_EQ) { // if overridden, emit the generic version return Some(false); } @@ -3556,8 +3683,6 @@ fn gen_equality_specialized( jit_guard_known_klass( jit, asm, - ocb, - unsafe { rb_cString }, a_opnd, a_opnd.into(), comptime_a, @@ -3569,7 +3694,7 @@ fn gen_equality_specialized( let ret = asm.new_label("ret"); // Spill for ccall. For safety, unconditionally spill temps before branching. - asm.spill_temps(); + asm.spill_regs(); // If they are equal by identity, return true asm.cmp(a_opnd, b_opnd); @@ -3583,8 +3708,6 @@ fn gen_equality_specialized( jit_guard_known_klass( jit, asm, - ocb, - unsafe { rb_cString }, b_opnd, b_opnd.into(), comptime_b, @@ -3619,54 +3742,48 @@ fn gen_equality_specialized( fn gen_opt_eq( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - let specialized = match gen_equality_specialized(jit, asm, ocb, true) { + let specialized = match gen_equality_specialized(jit, asm, true) { Some(specialized) => specialized, None => { // Defer compilation so we can specialize base on a runtime receiver - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + return jit.defer_compilation(asm); } }; if specialized { - jump_to_next_insn(jit, asm, ocb); - Some(EndBlock) + jump_to_next_insn(jit, asm) } else { - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } fn gen_opt_neq( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // opt_neq is passed two rb_call_data as arguments: // first for ==, second for != let cd = jit.get_arg(1).as_ptr(); - perf_call! { gen_send_general(jit, asm, ocb, cd, None) } + perf_call! { gen_send_general(jit, asm, cd, None) } } fn gen_opt_aref( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let cd: *const rb_call_data = jit.get_arg(0).as_ptr(); let argc = unsafe { vm_ci_argc((*cd).ci) }; // Only JIT one arg calls like `ary[6]` if argc != 1 { - gen_counter_incr(asm, Counter::opt_aref_argc_not_one); + gen_counter_incr(jit, asm, Counter::opt_aref_argc_not_one); return None; } // Defer compilation so we can specialize base on a runtime receiver - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } // Specialize base on compile time values @@ -3674,7 +3791,7 @@ fn gen_opt_aref( let comptime_recv = jit.peek_at_stack(&asm.ctx, 1); if comptime_recv.class_of() == unsafe { rb_cArray } && comptime_idx.fixnum_p() { - if !assume_bop_not_redefined(jit, asm, ocb, ARRAY_REDEFINED_OP_FLAG, BOP_AREF) { + if !assume_bop_not_redefined(jit, asm, ARRAY_REDEFINED_OP_FLAG, BOP_AREF) { return None; } @@ -3687,8 +3804,6 @@ fn gen_opt_aref( jit_guard_known_klass( jit, asm, - ocb, - unsafe { rb_cArray }, recv_opnd, recv_opnd.into(), comptime_recv, @@ -3716,10 +3831,9 @@ fn gen_opt_aref( } // Jump to next instruction. This allows guard chains to share the same successor. - jump_to_next_insn(jit, asm, ocb); - return Some(EndBlock); + return jump_to_next_insn(jit, asm); } else if comptime_recv.class_of() == unsafe { rb_cHash } { - if !assume_bop_not_redefined(jit, asm, ocb, HASH_REDEFINED_OP_FLAG, BOP_AREF) { + if !assume_bop_not_redefined(jit, asm, HASH_REDEFINED_OP_FLAG, BOP_AREF) { return None; } @@ -3729,8 +3843,6 @@ fn gen_opt_aref( jit_guard_known_klass( jit, asm, - ocb, - unsafe { rb_cHash }, recv_opnd, recv_opnd.into(), comptime_recv, @@ -3754,23 +3866,20 @@ fn gen_opt_aref( asm.mov(stack_ret, val); // Jump to next instruction. This allows guard chains to share the same successor. - jump_to_next_insn(jit, asm, ocb); - Some(EndBlock) + jump_to_next_insn(jit, asm) } else { // General case. Call the [] method. - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } fn gen_opt_aset( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Defer compilation so we can specialize on a runtime `self` - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } let comptime_recv = jit.peek_at_stack(&asm.ctx, 2); @@ -3786,8 +3895,6 @@ fn gen_opt_aset( jit_guard_known_klass( jit, asm, - ocb, - unsafe { rb_cArray }, recv, recv.into(), comptime_recv, @@ -3799,8 +3906,6 @@ fn gen_opt_aset( jit_guard_known_klass( jit, asm, - ocb, - unsafe { rb_cInteger }, key, key.into(), comptime_key, @@ -3827,15 +3932,12 @@ fn gen_opt_aset( let stack_ret = asm.stack_push(Type::Unknown); asm.mov(stack_ret, val); - jump_to_next_insn(jit, asm, ocb); - return Some(EndBlock); + return jump_to_next_insn(jit, asm) } else if comptime_recv.class_of() == unsafe { rb_cHash } { // Guard receiver is a Hash jit_guard_known_klass( jit, asm, - ocb, - unsafe { rb_cHash }, recv, recv.into(), comptime_recv, @@ -3857,67 +3959,31 @@ fn gen_opt_aset( let stack_ret = asm.stack_push(Type::Unknown); asm.mov(stack_ret, ret); - jump_to_next_insn(jit, asm, ocb); - Some(EndBlock) + jump_to_next_insn(jit, asm) } else { - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } -fn gen_opt_aref_with( - jit: &mut JITState, - asm: &mut Assembler, - _ocb: &mut OutlinedCb, -) -> Option<CodegenStatus>{ - // We might allocate or raise - jit_prepare_non_leaf_call(jit, asm); - - let key_opnd = Opnd::Value(jit.get_arg(0)); - let recv_opnd = asm.stack_opnd(0); - - extern "C" { - fn rb_vm_opt_aref_with(recv: VALUE, key: VALUE) -> VALUE; - } - - let val_opnd = asm.ccall( - rb_vm_opt_aref_with as *const u8, - vec![ - recv_opnd, - key_opnd - ], - ); - asm.stack_pop(1); // Keep it on stack during GC - - asm.cmp(val_opnd, Qundef.into()); - asm.je(Target::side_exit(Counter::opt_aref_with_qundef)); - - let top = asm.stack_push(Type::Unknown); - asm.mov(top, val_opnd); - - return Some(KeepCompiling); -} - fn gen_opt_and( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let two_fixnums = match asm.ctx.two_fixnums_on_stack(jit) { Some(two_fixnums) => two_fixnums, None => { // Defer compilation so we can specialize on a runtime `self` - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + return jit.defer_compilation(asm); } }; if two_fixnums { - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_AND) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, BOP_AND) { return None; } // Check that both operands are fixnums - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); // Get the operands and destination from the stack let arg1 = asm.stack_pop(1); @@ -3933,31 +3999,29 @@ fn gen_opt_and( Some(KeepCompiling) } else { // Delegate to send, call the method on the recv - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } fn gen_opt_or( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let two_fixnums = match asm.ctx.two_fixnums_on_stack(jit) { Some(two_fixnums) => two_fixnums, None => { // Defer compilation so we can specialize on a runtime `self` - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + return jit.defer_compilation(asm); } }; if two_fixnums { - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_OR) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, BOP_OR) { return None; } // Check that both operands are fixnums - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); // Get the operands and destination from the stack let arg1 = asm.stack_pop(1); @@ -3973,31 +4037,29 @@ fn gen_opt_or( Some(KeepCompiling) } else { // Delegate to send, call the method on the recv - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } fn gen_opt_minus( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let two_fixnums = match asm.ctx.two_fixnums_on_stack(jit) { Some(two_fixnums) => two_fixnums, None => { // Defer compilation so we can specialize on a runtime `self` - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + return jit.defer_compilation(asm); } }; if two_fixnums { - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) { return None; } // Check that both operands are fixnums - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); // Get the operands and destination from the stack let arg1 = asm.stack_pop(1); @@ -4015,31 +4077,29 @@ fn gen_opt_minus( Some(KeepCompiling) } else { // Delegate to send, call the method on the recv - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } fn gen_opt_mult( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let two_fixnums = match asm.ctx.two_fixnums_on_stack(jit) { Some(two_fixnums) => two_fixnums, None => { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + return jit.defer_compilation(asm); } }; // Fallback to a method call if it overflows if two_fixnums && asm.ctx.get_chain_depth() == 0 { - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_MULT) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, BOP_MULT) { return None; } // Check that both operands are fixnums - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); // Get the operands from the stack let arg1 = asm.stack_pop(1); @@ -4050,7 +4110,7 @@ fn gen_opt_mult( let arg0_untag = asm.rshift(arg0, Opnd::UImm(1)); let arg1_untag = asm.sub(arg1, Opnd::UImm(1)); let out_val = asm.mul(arg0_untag, arg1_untag); - jit_chain_guard(JCC_JO_MUL, jit, asm, ocb, 1, Counter::opt_mult_overflow); + jit_chain_guard(JCC_JO_MUL, jit, asm, 1, Counter::opt_mult_overflow); let out_val = asm.add(out_val, Opnd::UImm(1)); // Push the output on the stack @@ -4059,40 +4119,37 @@ fn gen_opt_mult( Some(KeepCompiling) } else { - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } fn gen_opt_div( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Delegate to send, call the method on the recv - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } fn gen_opt_mod( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let two_fixnums = match asm.ctx.two_fixnums_on_stack(jit) { Some(two_fixnums) => two_fixnums, None => { // Defer compilation so we can specialize on a runtime `self` - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + return jit.defer_compilation(asm); } }; if two_fixnums { - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_MOD) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, BOP_MOD) { return None; } // Check that both operands are fixnums - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); // Get the operands and destination from the stack let arg1 = asm.stack_pop(1); @@ -4113,52 +4170,47 @@ fn gen_opt_mod( Some(KeepCompiling) } else { // Delegate to send, call the method on the recv - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } } fn gen_opt_ltlt( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Delegate to send, call the method on the recv - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } fn gen_opt_nil_p( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Delegate to send, call the method on the recv - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } fn gen_opt_empty_p( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Delegate to send, call the method on the recv - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } fn gen_opt_succ( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Delegate to send, call the method on the recv - gen_opt_send_without_block(jit, asm, ocb) + gen_opt_send_without_block(jit, asm) } fn gen_opt_str_freeze( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - if !assume_bop_not_redefined(jit, asm, ocb, STRING_REDEFINED_OP_FLAG, BOP_FREEZE) { + if !assume_bop_not_redefined(jit, asm, STRING_REDEFINED_OP_FLAG, BOP_FREEZE) { return None; } @@ -4171,12 +4223,45 @@ fn gen_opt_str_freeze( Some(KeepCompiling) } +fn gen_opt_ary_freeze( + jit: &mut JITState, + asm: &mut Assembler, +) -> Option<CodegenStatus> { + if !assume_bop_not_redefined(jit, asm, ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) { + return None; + } + + let ary = jit.get_arg(0); + + // Push the return value onto the stack + let stack_ret = asm.stack_push(Type::CArray); + asm.mov(stack_ret, ary.into()); + + Some(KeepCompiling) +} + +fn gen_opt_hash_freeze( + jit: &mut JITState, + asm: &mut Assembler, +) -> Option<CodegenStatus> { + if !assume_bop_not_redefined(jit, asm, HASH_REDEFINED_OP_FLAG, BOP_FREEZE) { + return None; + } + + let hash = jit.get_arg(0); + + // Push the return value onto the stack + let stack_ret = asm.stack_push(Type::CHash); + asm.mov(stack_ret, hash.into()); + + Some(KeepCompiling) +} + fn gen_opt_str_uminus( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - if !assume_bop_not_redefined(jit, asm, ocb, STRING_REDEFINED_OP_FLAG, BOP_UMINUS) { + if !assume_bop_not_redefined(jit, asm, STRING_REDEFINED_OP_FLAG, BOP_UMINUS) { return None; } @@ -4192,7 +4277,6 @@ fn gen_opt_str_uminus( fn gen_opt_newarray_max( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let num = jit.get_arg(0).as_u32(); @@ -4222,53 +4306,109 @@ fn gen_opt_newarray_max( Some(KeepCompiling) } -fn gen_opt_newarray_send( +fn gen_opt_duparray_send( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let method = jit.get_arg(1).as_u64(); - if method == ID!(min) { - gen_opt_newarray_min(jit, asm, _ocb) - } else if method == ID!(max) { - gen_opt_newarray_max(jit, asm, _ocb) - } else if method == ID!(hash) { - gen_opt_newarray_hash(jit, asm, _ocb) - } else if method == ID!(pack) { - gen_opt_newarray_pack(jit, asm, _ocb) + if method == ID!(include_p) { + gen_opt_duparray_send_include_p(jit, asm) } else { None } } -fn gen_opt_newarray_pack( +fn gen_opt_duparray_send_include_p( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - // num == 4 ( for this code ) + asm_comment!(asm, "opt_duparray_send include_p"); + + let ary = jit.get_arg(0); + let argc = jit.get_arg(2).as_usize(); + + // Save the PC and SP because we may call #include? + jit_prepare_non_leaf_call(jit, asm); + + extern "C" { + fn rb_vm_opt_duparray_include_p(ec: EcPtr, ary: VALUE, target: VALUE) -> VALUE; + } + + let target = asm.ctx.sp_opnd(-1); + + let val_opnd = asm.ccall( + rb_vm_opt_duparray_include_p as *const u8, + vec![ + EC, + ary.into(), + target, + ], + ); + + asm.stack_pop(argc); + let stack_ret = asm.stack_push(Type::Unknown); + asm.mov(stack_ret, val_opnd); + + Some(KeepCompiling) +} + +fn gen_opt_newarray_send( + jit: &mut JITState, + asm: &mut Assembler, +) -> Option<CodegenStatus> { + let method = jit.get_arg(1).as_u32(); + + if method == VM_OPT_NEWARRAY_SEND_MIN { + gen_opt_newarray_min(jit, asm) + } else if method == VM_OPT_NEWARRAY_SEND_MAX { + gen_opt_newarray_max(jit, asm) + } else if method == VM_OPT_NEWARRAY_SEND_HASH { + gen_opt_newarray_hash(jit, asm) + } else if method == VM_OPT_NEWARRAY_SEND_INCLUDE_P { + gen_opt_newarray_include_p(jit, asm) + } else if method == VM_OPT_NEWARRAY_SEND_PACK { + gen_opt_newarray_pack_buffer(jit, asm, 1, None) + } else if method == VM_OPT_NEWARRAY_SEND_PACK_BUFFER { + gen_opt_newarray_pack_buffer(jit, asm, 2, Some(1)) + } else { + None + } +} + +fn gen_opt_newarray_pack_buffer( + jit: &mut JITState, + asm: &mut Assembler, + fmt_offset: u32, + buffer: Option<u32>, +) -> Option<CodegenStatus> { + asm_comment!(asm, "opt_newarray_send pack"); + let num = jit.get_arg(0).as_u32(); // Save the PC and SP because we may call #pack jit_prepare_non_leaf_call(jit, asm); extern "C" { - fn rb_vm_opt_newarray_pack(ec: EcPtr, num: u32, elts: *const VALUE, fmt: VALUE) -> VALUE; + fn rb_vm_opt_newarray_pack_buffer(ec: EcPtr, num: u32, elts: *const VALUE, fmt: VALUE, buffer: VALUE) -> VALUE; } let values_opnd = asm.ctx.sp_opnd(-(num as i32)); let values_ptr = asm.lea(values_opnd); - let fmt_string = asm.ctx.sp_opnd(-1); + let fmt_string = asm.ctx.sp_opnd(-(fmt_offset as i32)); let val_opnd = asm.ccall( - rb_vm_opt_newarray_pack as *const u8, + rb_vm_opt_newarray_pack_buffer as *const u8, vec![ EC, - (num - 1).into(), + (num - fmt_offset).into(), values_ptr, - fmt_string + fmt_string, + match buffer { + None => Qundef.into(), + Some(i) => asm.ctx.sp_opnd(-(i as i32)), + }, ], ); @@ -4282,7 +4422,6 @@ fn gen_opt_newarray_pack( fn gen_opt_newarray_hash( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let num = jit.get_arg(0).as_u32(); @@ -4313,10 +4452,45 @@ fn gen_opt_newarray_hash( Some(KeepCompiling) } +fn gen_opt_newarray_include_p( + jit: &mut JITState, + asm: &mut Assembler, +) -> Option<CodegenStatus> { + asm_comment!(asm, "opt_newarray_send include?"); + + let num = jit.get_arg(0).as_u32(); + + // Save the PC and SP because we may call customized methods. + jit_prepare_non_leaf_call(jit, asm); + + extern "C" { + fn rb_vm_opt_newarray_include_p(ec: EcPtr, num: u32, elts: *const VALUE, target: VALUE) -> VALUE; + } + + let values_opnd = asm.ctx.sp_opnd(-(num as i32)); + let values_ptr = asm.lea(values_opnd); + let target = asm.ctx.sp_opnd(-1); + + let val_opnd = asm.ccall( + rb_vm_opt_newarray_include_p as *const u8, + vec![ + EC, + (num - 1).into(), + values_ptr, + target + ], + ); + + asm.stack_pop(num.as_usize()); + let stack_ret = asm.stack_push(Type::Unknown); + asm.mov(stack_ret, val_opnd); + + Some(KeepCompiling) +} + fn gen_opt_newarray_min( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let num = jit.get_arg(0).as_u32(); @@ -4350,39 +4524,34 @@ fn gen_opt_newarray_min( fn gen_opt_not( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - return gen_opt_send_without_block(jit, asm, ocb); + return gen_opt_send_without_block(jit, asm); } fn gen_opt_size( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - return gen_opt_send_without_block(jit, asm, ocb); + return gen_opt_send_without_block(jit, asm); } fn gen_opt_length( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - return gen_opt_send_without_block(jit, asm, ocb); + return gen_opt_send_without_block(jit, asm); } fn gen_opt_regexpmatch2( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - return gen_opt_send_without_block(jit, asm, ocb); + return gen_opt_send_without_block(jit, asm); } fn gen_opt_case_dispatch( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Normally this instruction would lookup the key in a hash and jump to an // offset based on that. @@ -4391,9 +4560,8 @@ fn gen_opt_case_dispatch( // We'd hope that our jitted code will be sufficiently fast without the // hash lookup, at least for small hashes, but it's worth revisiting this // assumption in the future. - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } let case_hash = jit.get_arg(0); @@ -4426,11 +4594,11 @@ fn gen_opt_case_dispatch( // If megamorphic, fallback to compiling branch instructions after opt_case_dispatch let megamorphic = asm.ctx.get_chain_depth() >= CASE_WHEN_MAX_DEPTH; if megamorphic { - gen_counter_incr(asm, Counter::num_opt_case_dispatch_megamorphic); + gen_counter_incr(jit, asm, Counter::num_opt_case_dispatch_megamorphic); } if comptime_key.fixnum_p() && comptime_key.0 <= u32::MAX.as_usize() && case_hash_all_fixnum_p(case_hash) && !megamorphic { - if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_EQQ) { + if !assume_bop_not_redefined(jit, asm, INTEGER_REDEFINED_OP_FLAG, BOP_EQQ) { return None; } @@ -4440,7 +4608,6 @@ fn gen_opt_case_dispatch( JCC_JNE, jit, asm, - ocb, CASE_WHEN_MAX_DEPTH, Counter::opt_case_dispatch_megamorphic, ); @@ -4470,7 +4637,6 @@ fn gen_opt_case_dispatch( fn gen_branchif( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let jump_offset = jit.get_arg(0).as_i32(); @@ -4507,10 +4673,8 @@ fn gen_branchif( // Generate the branch instructions let ctx = asm.ctx; - gen_branch( - jit, + jit.gen_branch( asm, - ocb, jump_block, &ctx, Some(next_block), @@ -4525,7 +4689,6 @@ fn gen_branchif( fn gen_branchunless( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let jump_offset = jit.get_arg(0).as_i32(); @@ -4563,10 +4726,8 @@ fn gen_branchunless( // Generate the branch instructions let ctx = asm.ctx; - gen_branch( - jit, + jit.gen_branch( asm, - ocb, jump_block, &ctx, Some(next_block), @@ -4581,7 +4742,6 @@ fn gen_branchunless( fn gen_branchnil( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let jump_offset = jit.get_arg(0).as_i32(); @@ -4616,10 +4776,8 @@ fn gen_branchnil( asm.cmp(val_opnd, Opnd::UImm(Qnil.into())); // Generate the branch instructions let ctx = asm.ctx; - gen_branch( - jit, + jit.gen_branch( asm, - ocb, jump_block, &ctx, Some(next_block), @@ -4634,18 +4792,17 @@ fn gen_branchnil( fn gen_throw( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let throw_state = jit.get_arg(0).as_u64(); let throwobj = asm.stack_pop(1); let throwobj = asm.load(throwobj); // Gather some statistics about throw - gen_counter_incr(asm, Counter::num_throw); + gen_counter_incr(jit, asm, Counter::num_throw); match (throw_state & VM_THROW_STATE_MASK as u64) as u32 { - RUBY_TAG_BREAK => gen_counter_incr(asm, Counter::num_throw_break), - RUBY_TAG_RETRY => gen_counter_incr(asm, Counter::num_throw_retry), - RUBY_TAG_RETURN => gen_counter_incr(asm, Counter::num_throw_return), + RUBY_TAG_BREAK => gen_counter_incr(jit, asm, Counter::num_throw_break), + RUBY_TAG_RETRY => gen_counter_incr(jit, asm, Counter::num_throw_retry), + RUBY_TAG_RETURN => gen_counter_incr(jit, asm, Counter::num_throw_return), _ => {}, } @@ -4674,10 +4831,72 @@ fn gen_throw( Some(EndBlock) } +fn gen_opt_new( + jit: &mut JITState, + asm: &mut Assembler, +) -> Option<CodegenStatus> { + let cd = jit.get_arg(0).as_ptr(); + let jump_offset = jit.get_arg(1).as_i32(); + + if !jit.at_compile_target() { + return jit.defer_compilation(asm); + } + + let ci = unsafe { get_call_data_ci(cd) }; // info about the call site + let mid = unsafe { vm_ci_mid(ci) }; + let argc: i32 = unsafe { vm_ci_argc(ci) }.try_into().unwrap(); + + let recv_idx = argc; + let comptime_recv = jit.peek_at_stack(&asm.ctx, recv_idx as isize); + + // This is a singleton class + let comptime_recv_klass = comptime_recv.class_of(); + + let recv = asm.stack_opnd(recv_idx); + + perf_call!("opt_new: ", jit_guard_known_klass( + jit, + asm, + recv, + recv.into(), + comptime_recv, + SEND_MAX_DEPTH, + Counter::guard_send_klass_megamorphic, + )); + + // We now know that it's always comptime_recv_klass + if jit.assume_expected_cfunc(asm, comptime_recv_klass, mid, rb_class_new_instance_pass_kw as _) { + // Fast path + // call rb_class_alloc to actually allocate + jit_prepare_non_leaf_call(jit, asm); + let obj = asm.ccall(rb_obj_alloc as _, vec![comptime_recv.into()]); + + // Get a reference to the stack location where we need to save the + // return instance. + let result = asm.stack_opnd(recv_idx + 1); + let recv = asm.stack_opnd(recv_idx); + + // Replace the receiver for the upcoming initialize call + asm.ctx.set_opnd_mapping(recv.into(), TempMapping::MapToStack(Type::UnknownHeap)); + asm.mov(recv, obj); + + // Save the allocated object for return + asm.ctx.set_opnd_mapping(result.into(), TempMapping::MapToStack(Type::UnknownHeap)); + asm.mov(result, obj); + + jump_to_next_insn(jit, asm) + } else { + // general case + + // Get the branch target instruction offsets + let jump_idx = jit.next_insn_idx() as i32 + jump_offset; + return end_block_with_jump(jit, asm, jump_idx as u16); + } +} + fn gen_jump( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let jump_offset = jit.get_arg(0).as_i32(); @@ -4708,21 +4927,20 @@ fn gen_jump( fn jit_guard_known_klass( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, - known_klass: VALUE, obj_opnd: Opnd, insn_opnd: YARVOpnd, sample_instance: VALUE, max_chain_depth: u8, counter: Counter, ) { + let known_klass = sample_instance.class_of(); let val_type = asm.ctx.get_opnd_type(insn_opnd); if val_type.known_class() == Some(known_klass) { // Unless frozen, Array, Hash, and String objects may change their RBASIC_CLASS // when they get a singleton class. Those types need invalidations. if unsafe { [rb_cArray, rb_cHash, rb_cString].contains(&known_klass) } { - if jit.assume_no_singleton_class(asm, ocb, known_klass) { + if jit.assume_no_singleton_class(asm, known_klass) { // Speculate that this object will not have a singleton class, // and invalidate the block in case it does. return; @@ -4739,7 +4957,7 @@ fn jit_guard_known_klass( asm_comment!(asm, "guard object is nil"); asm.cmp(obj_opnd, Qnil.into()); - jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JNE, jit, asm, max_chain_depth, counter); asm.ctx.upgrade_opnd_type(insn_opnd, Type::Nil); } else if unsafe { known_klass == rb_cTrueClass } { @@ -4748,7 +4966,7 @@ fn jit_guard_known_klass( asm_comment!(asm, "guard object is true"); asm.cmp(obj_opnd, Qtrue.into()); - jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JNE, jit, asm, max_chain_depth, counter); asm.ctx.upgrade_opnd_type(insn_opnd, Type::True); } else if unsafe { known_klass == rb_cFalseClass } { @@ -4758,7 +4976,7 @@ fn jit_guard_known_klass( asm_comment!(asm, "guard object is false"); assert!(Qfalse.as_i32() == 0); asm.test(obj_opnd, obj_opnd); - jit_chain_guard(JCC_JNZ, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JNZ, jit, asm, max_chain_depth, counter); asm.ctx.upgrade_opnd_type(insn_opnd, Type::False); } else if unsafe { known_klass == rb_cInteger } && sample_instance.fixnum_p() { @@ -4768,7 +4986,7 @@ fn jit_guard_known_klass( asm_comment!(asm, "guard object is fixnum"); asm.test(obj_opnd, Opnd::Imm(RUBY_FIXNUM_FLAG as i64)); - jit_chain_guard(JCC_JZ, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JZ, jit, asm, max_chain_depth, counter); asm.ctx.upgrade_opnd_type(insn_opnd, Type::Fixnum); } else if unsafe { known_klass == rb_cSymbol } && sample_instance.static_sym_p() { assert!(!val_type.is_heap()); @@ -4780,7 +4998,7 @@ fn jit_guard_known_klass( asm_comment!(asm, "guard object is static symbol"); assert!(RUBY_SPECIAL_SHIFT == 8); asm.cmp(obj_opnd.with_num_bits(8).unwrap(), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); - jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JNE, jit, asm, max_chain_depth, counter); asm.ctx.upgrade_opnd_type(insn_opnd, Type::ImmSymbol); } } else if unsafe { known_klass == rb_cFloat } && sample_instance.flonum_p() { @@ -4792,7 +5010,7 @@ fn jit_guard_known_klass( asm_comment!(asm, "guard object is flonum"); let flag_bits = asm.and(obj_opnd, Opnd::UImm(RUBY_FLONUM_MASK as u64)); asm.cmp(flag_bits, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); - jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JNE, jit, asm, max_chain_depth, counter); asm.ctx.upgrade_opnd_type(insn_opnd, Type::Flonum); } } else if unsafe { @@ -4814,23 +5032,23 @@ fn jit_guard_known_klass( // IO#reopen can be used to change the class and singleton class of IO objects! asm_comment!(asm, "guard known object with singleton class"); asm.cmp(obj_opnd, sample_instance.into()); - jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JNE, jit, asm, max_chain_depth, counter); } else if val_type == Type::CString && unsafe { known_klass == rb_cString } { // guard elided because the context says we've already checked unsafe { assert_eq!(sample_instance.class_of(), rb_cString, "context says class is exactly ::String") }; } else { - assert!(!val_type.is_imm()); + assert!(!val_type.is_imm(), "{insn_opnd:?} should be a heap object, but was {val_type:?} for {sample_instance:?}"); // Check that the receiver is a heap object // Note: if we get here, the class doesn't have immediate instances. if !val_type.is_heap() { asm_comment!(asm, "guard not immediate"); asm.test(obj_opnd, (RUBY_IMMEDIATE_MASK as u64).into()); - jit_chain_guard(JCC_JNZ, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JNZ, jit, asm, max_chain_depth, counter); asm.cmp(obj_opnd, Qfalse.into()); - jit_chain_guard(JCC_JE, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JE, jit, asm, max_chain_depth, counter); asm.ctx.upgrade_opnd_type(insn_opnd, Type::UnknownHeap); } @@ -4846,7 +5064,7 @@ fn jit_guard_known_klass( // TODO: jit_mov_gc_ptr keeps a strong reference, which leaks the class. asm_comment!(asm, "guard known class"); asm.cmp(klass_opnd, known_klass.into()); - jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); + jit_chain_guard(JCC_JNE, jit, asm, max_chain_depth, counter); if known_klass == unsafe { rb_cString } { asm.ctx.upgrade_opnd_type(insn_opnd, Type::CString); @@ -4886,7 +5104,6 @@ fn jit_protected_callee_ancestry_guard( fn jit_rb_obj_not( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -4921,7 +5138,6 @@ fn jit_rb_obj_not( fn jit_rb_true( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -4939,7 +5155,6 @@ fn jit_rb_true( fn jit_rb_false( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -4957,7 +5172,6 @@ fn jit_rb_false( fn jit_rb_kernel_is_a( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5010,7 +5224,6 @@ fn jit_rb_kernel_is_a( fn jit_rb_kernel_instance_of( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5055,7 +5268,6 @@ fn jit_rb_kernel_instance_of( JCC_JNE, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::guard_send_instance_of_class_mismatch, ); @@ -5075,7 +5287,6 @@ fn jit_rb_kernel_instance_of( fn jit_rb_mod_eqq( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5104,12 +5315,38 @@ fn jit_rb_mod_eqq( return true; } +// Substitution for rb_mod_name(). Returns the name of a module/class. +fn jit_rb_mod_name( + _jit: &mut JITState, + asm: &mut Assembler, + _ci: *const rb_callinfo, + _cme: *const rb_callable_method_entry_t, + _block: Option<BlockHandler>, + argc: i32, + _known_recv_class: Option<VALUE>, +) -> bool { + if argc != 0 { + return false; + } + + asm_comment!(asm, "Module#name"); + + // rb_mod_name() never allocates, so no preparation needed. + let name = asm.ccall(rb_mod_name as _, vec![asm.stack_opnd(0)]); + + let _ = asm.stack_pop(1); // pop self + // call-seq: mod.name -> string or nil + let ret = asm.stack_push(Type::Unknown); + asm.mov(ret, name); + + true +} + // Codegen for rb_obj_equal() // object identity comparison fn jit_rb_obj_equal( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5133,21 +5370,19 @@ fn jit_rb_obj_equal( fn jit_rb_obj_not_equal( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, _argc: i32, _known_recv_class: Option<VALUE>, ) -> bool { - gen_equality_specialized(jit, asm, ocb, false) == Some(true) + gen_equality_specialized(jit, asm, false) == Some(true) } // Codegen for rb_int_equal() fn jit_rb_int_equal( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5155,7 +5390,7 @@ fn jit_rb_int_equal( _known_recv_class: Option<VALUE>, ) -> bool { // Check that both operands are fixnums - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); // Compare the arguments asm_comment!(asm, "rb_int_equal"); @@ -5172,7 +5407,6 @@ fn jit_rb_int_equal( fn jit_rb_int_succ( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5199,10 +5433,38 @@ fn jit_rb_int_succ( true } +fn jit_rb_int_pred( + _jit: &mut JITState, + asm: &mut Assembler, + _ci: *const rb_callinfo, + _cme: *const rb_callable_method_entry_t, + _block: Option<BlockHandler>, + _argc: i32, + _known_recv_class: Option<VALUE>, +) -> bool { + // Guard the receiver is fixnum + let recv_type = asm.ctx.get_opnd_type(StackOpnd(0)); + let recv = asm.stack_pop(1); + if recv_type != Type::Fixnum { + asm_comment!(asm, "guard object is fixnum"); + asm.test(recv, Opnd::Imm(RUBY_FIXNUM_FLAG as i64)); + asm.jz(Target::side_exit(Counter::send_pred_not_fixnum)); + } + + asm_comment!(asm, "Integer#pred"); + let out_val = asm.sub(recv, Opnd::Imm(2)); // 2 is untagged Fixnum 1 + asm.jo(Target::side_exit(Counter::send_pred_underflow)); + + // Push the output onto the stack + let dst = asm.stack_push(Type::Fixnum); + asm.mov(dst, out_val); + + true +} + fn jit_rb_int_div( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5212,7 +5474,7 @@ fn jit_rb_int_div( if asm.ctx.two_fixnums_on_stack(jit) != Some(true) { return false; } - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); // rb_fix_div_fix may GC-allocate for Bignum jit_prepare_call_with_gc(jit, asm); @@ -5236,7 +5498,6 @@ fn jit_rb_int_div( fn jit_rb_int_lshift( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5246,7 +5507,7 @@ fn jit_rb_int_lshift( if asm.ctx.two_fixnums_on_stack(jit) != Some(true) { return false; } - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); let comptime_shift = jit.peek_at_stack(&asm.ctx, 0); @@ -5276,7 +5537,6 @@ fn jit_rb_int_lshift( JCC_JNE, jit, asm, - ocb, 1, Counter::lshift_amount_changed, ); @@ -5305,7 +5565,6 @@ fn fixnum_left_shift_body(asm: &mut Assembler, lhs: Opnd, shift_amt: u64) { fn jit_rb_int_rshift( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5315,7 +5574,7 @@ fn jit_rb_int_rshift( if asm.ctx.two_fixnums_on_stack(jit) != Some(true) { return false; } - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); let comptime_shift = jit.peek_at_stack(&asm.ctx, 0); @@ -5341,7 +5600,6 @@ fn jit_rb_int_rshift( JCC_JNE, jit, asm, - ocb, 1, Counter::rshift_amount_changed, ); @@ -5358,7 +5616,6 @@ fn jit_rb_int_rshift( fn jit_rb_int_xor( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5368,7 +5625,7 @@ fn jit_rb_int_xor( if asm.ctx.two_fixnums_on_stack(jit) != Some(true) { return false; } - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); let rhs = asm.stack_pop(1); let lhs = asm.stack_pop(1); @@ -5385,7 +5642,6 @@ fn jit_rb_int_xor( fn jit_rb_int_aref( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5398,7 +5654,7 @@ fn jit_rb_int_aref( if asm.ctx.two_fixnums_on_stack(jit) != Some(true) { return false; } - guard_two_fixnums(jit, asm, ocb); + guard_two_fixnums(jit, asm); asm_comment!(asm, "Integer#[]"); let obj = asm.stack_pop(1); @@ -5414,7 +5670,6 @@ fn jit_rb_int_aref( fn jit_rb_float_plus( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5428,8 +5683,6 @@ fn jit_rb_float_plus( jit_guard_known_klass( jit, asm, - ocb, - comptime_obj.class_of(), obj, obj.into(), comptime_obj, @@ -5458,7 +5711,6 @@ fn jit_rb_float_plus( fn jit_rb_float_minus( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5472,8 +5724,6 @@ fn jit_rb_float_minus( jit_guard_known_klass( jit, asm, - ocb, - comptime_obj.class_of(), obj, obj.into(), comptime_obj, @@ -5502,7 +5752,6 @@ fn jit_rb_float_minus( fn jit_rb_float_mul( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5516,8 +5765,6 @@ fn jit_rb_float_mul( jit_guard_known_klass( jit, asm, - ocb, - comptime_obj.class_of(), obj, obj.into(), comptime_obj, @@ -5546,7 +5793,6 @@ fn jit_rb_float_mul( fn jit_rb_float_div( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5560,8 +5806,6 @@ fn jit_rb_float_div( jit_guard_known_klass( jit, asm, - ocb, - comptime_obj.class_of(), obj, obj.into(), comptime_obj, @@ -5591,7 +5835,6 @@ fn jit_rb_float_div( fn jit_rb_str_uplus( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5605,7 +5848,7 @@ fn jit_rb_str_uplus( // We allocate when we dup the string jit_prepare_call_with_gc(jit, asm); - asm.spill_temps(); // For ccall. Unconditionally spill them for RegTemps consistency. + asm.spill_regs(); // For ccall. Unconditionally spill them for RegMappings consistency. asm_comment!(asm, "Unary plus on string"); let recv_opnd = asm.stack_pop(1); @@ -5623,7 +5866,7 @@ fn jit_rb_str_uplus( asm.jz(ret_label); // Str is frozen - duplicate it - asm.spill_temps(); // for ccall + asm.spill_regs(); // for ccall let ret_opnd = asm.ccall(rb_str_dup as *const u8, vec![recv_opnd]); asm.mov(stack_ret, ret_opnd); @@ -5635,7 +5878,6 @@ fn jit_rb_str_uplus( fn jit_rb_str_length( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5662,7 +5904,6 @@ fn jit_rb_str_length( fn jit_rb_str_bytesize( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5694,7 +5935,6 @@ fn jit_rb_str_bytesize( fn jit_rb_str_byteslice( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5733,10 +5973,85 @@ fn jit_rb_str_byteslice( true } +fn jit_rb_str_aref_m( + jit: &mut JITState, + asm: &mut Assembler, + _ci: *const rb_callinfo, + _cme: *const rb_callable_method_entry_t, + _block: Option<BlockHandler>, + argc: i32, + _known_recv_class: Option<VALUE>, +) -> bool { + // In yjit-bench the most common usages by far are single fixnum or two fixnums. + // rb_str_substr should be leaf if indexes are fixnums + if argc == 2 { + match (asm.ctx.get_opnd_type(StackOpnd(0)), asm.ctx.get_opnd_type(StackOpnd(1))) { + (Type::Fixnum, Type::Fixnum) => {}, + // There is a two-argument form of (RegExp, Fixnum) which needs a different c func. + // Other types will raise. + _ => { return false }, + } + } else if argc == 1 { + match asm.ctx.get_opnd_type(StackOpnd(0)) { + Type::Fixnum => {}, + // Besides Fixnum this could also be a Range or a RegExp which are handled by separate c funcs. + // Other types will raise. + _ => { + // If the context doesn't have the type info we try a little harder. + let comptime_arg = jit.peek_at_stack(&asm.ctx, 0); + let arg0 = asm.stack_opnd(0); + if comptime_arg.fixnum_p() { + asm.test(arg0, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); + + jit_chain_guard( + JCC_JZ, + jit, + asm, + SEND_MAX_DEPTH, + Counter::guard_send_str_aref_not_fixnum, + ); + } else { + return false + } + }, + } + } else { + return false + } + + asm_comment!(asm, "String#[]"); + + // rb_str_substr allocates a substring + jit_prepare_call_with_gc(jit, asm); + + // Get stack operands after potential SP change + + // The "empty" arg distinguishes between the normal "one arg" behavior + // and the "two arg" special case that returns an empty string + // when the begin index is the length of the string. + // See the usages of rb_str_substr in string.c for more information. + let (beg_idx, empty, len) = if argc == 2 { + (1, Opnd::Imm(1), asm.stack_opnd(0)) + } else { + // If there is only one arg, the length will be 1. + (0, Opnd::Imm(0), VALUE::fixnum_from_usize(1).into()) + }; + + let beg = asm.stack_opnd(beg_idx); + let recv = asm.stack_opnd(beg_idx + 1); + + let ret_opnd = asm.ccall(rb_str_substr_two_fixnums as *const u8, vec![recv, beg, len, empty]); + asm.stack_pop(beg_idx as usize + 2); + + let out_opnd = asm.stack_push(Type::Unknown); + asm.mov(out_opnd, ret_opnd); + + true +} + fn jit_rb_str_getbyte( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5754,8 +6069,6 @@ fn jit_rb_str_getbyte( jit_guard_known_klass( jit, asm, - ocb, - comptime_idx.class_of(), idx, idx.into(), comptime_idx, @@ -5781,7 +6094,7 @@ fn jit_rb_str_getbyte( RUBY_OFFSET_RSTRING_LEN as i32, ); - // Exit if the indes is out of bounds + // Exit if the index is out of bounds asm.cmp(idx, str_len_opnd); asm.jge(Target::side_exit(Counter::getbyte_idx_out_of_bounds)); @@ -5808,7 +6121,6 @@ fn jit_rb_str_getbyte( fn jit_rb_str_setbyte( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5841,7 +6153,6 @@ fn jit_rb_str_setbyte( fn jit_rb_str_to_s( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5857,11 +6168,45 @@ fn jit_rb_str_to_s( false } +fn jit_rb_str_dup( + jit: &mut JITState, + asm: &mut Assembler, + _ci: *const rb_callinfo, + _cme: *const rb_callable_method_entry_t, + _block: Option<BlockHandler>, + _argc: i32, + known_recv_class: Option<VALUE>, +) -> bool { + // We specialize only the BARE_STRING_P case. Otherwise it's not leaf. + if unsafe { known_recv_class != Some(rb_cString) } { + return false; + } + asm_comment!(asm, "String#dup"); + + jit_prepare_call_with_gc(jit, asm); + + let recv_opnd = asm.stack_opnd(0); + let recv_opnd = asm.load(recv_opnd); + + let shape_id_offset = unsafe { rb_shape_id_offset() }; + let shape_opnd = Opnd::mem(64, recv_opnd, shape_id_offset); + asm.test(shape_opnd, Opnd::UImm(SHAPE_ID_HAS_IVAR_MASK as u64)); + asm.jnz(Target::side_exit(Counter::send_str_dup_exivar)); + + // Call rb_str_dup + let ret_opnd = asm.ccall(rb_str_dup as *const u8, vec![recv_opnd]); + + asm.stack_pop(1); + let stack_ret = asm.stack_push(Type::CString); + asm.mov(stack_ret, ret_opnd); + + true +} + // Codegen for rb_str_empty_p() fn jit_rb_str_empty_p( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5885,24 +6230,59 @@ fn jit_rb_str_empty_p( return true; } -// Codegen for rb_str_concat() -- *not* String#concat -// Frequently strings are concatenated using "out_str << next_str". -// This is common in Erb and similar templating languages. -fn jit_rb_str_concat( +// Codegen for rb_str_concat() with an integer argument -- *not* String#concat +// Using strings as a byte buffer often includes appending byte values to the end of the string. +fn jit_rb_str_concat_codepoint( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, _argc: i32, _known_recv_class: Option<VALUE>, ) -> bool { + asm_comment!(asm, "String#<< with codepoint argument"); + + // Either of the string concatenation functions we call will reallocate the string to grow its + // capacity if necessary. In extremely rare cases (i.e., string exceeds `LONG_MAX` bytes), + // either of the called functions will raise an exception. + jit_prepare_non_leaf_call(jit, asm); + + let codepoint = asm.stack_opnd(0); + let recv = asm.stack_opnd(1); + + guard_object_is_fixnum(jit, asm, codepoint, StackOpnd(0)); + + asm.ccall(rb_jit_str_concat_codepoint as *const u8, vec![recv, codepoint]); + + // The receiver is the return value, so we only need to pop the codepoint argument off the stack. + // We can reuse the receiver slot in the stack as the return value. + asm.stack_pop(1); + + true +} + +// Codegen for rb_str_concat() -- *not* String#concat +// Frequently strings are concatenated using "out_str << next_str". +// This is common in Erb and similar templating languages. +fn jit_rb_str_concat( + jit: &mut JITState, + asm: &mut Assembler, + ci: *const rb_callinfo, + cme: *const rb_callable_method_entry_t, + block: Option<BlockHandler>, + argc: i32, + known_recv_class: Option<VALUE>, +) -> bool { // The << operator can accept integer codepoints for characters // as the argument. We only specially optimise string arguments. // If the peeked-at compile time argument is something other than // a string, assume it won't be a string later either. let comptime_arg = jit.peek_at_stack(&asm.ctx, 0); + if unsafe { RB_TYPE_P(comptime_arg, RUBY_T_FIXNUM) } { + return jit_rb_str_concat_codepoint(jit, asm, ci, cme, block, argc, known_recv_class); + } + if ! unsafe { RB_TYPE_P(comptime_arg, RUBY_T_STRING) } { return false; } @@ -5914,7 +6294,14 @@ fn jit_rb_str_concat( // rb_str_buf_append may raise Encoding::CompatibilityError, but we accept compromised // backtraces on this method since the interpreter does the same thing on opt_ltlt. jit_prepare_non_leaf_call(jit, asm); - asm.spill_temps(); // For ccall. Unconditionally spill them for RegTemps consistency. + + // Explicitly spill temps before making any C calls. `ccall` will spill temps, but it does a + // check to only spill if it thinks it's necessary. That logic can't see through the runtime + // branching occurring in the code generated for this function. Consequently, the branch for + // the first `ccall` will spill registers but the second one will not. At run time, we may + // jump over that spill code when executing the second branch, leading situations that are + // quite hard to debug. If we spill up front we avoid diverging behavior. + asm.spill_regs(); let concat_arg = asm.stack_pop(1); let recv = asm.stack_pop(1); @@ -5947,7 +6334,7 @@ fn jit_rb_str_concat( // If encodings are different, use a slower encoding-aware concatenate asm.write_label(enc_mismatch); - asm.spill_temps(); // Ignore the register for the other local branch + asm.spill_regs(); // Ignore the register for the other local branch let ret_opnd = asm.ccall(rb_str_buf_append as *const u8, vec![recv, concat_arg]); let stack_ret = asm.stack_push(Type::TString); asm.mov(stack_ret, ret_opnd); @@ -5962,7 +6349,6 @@ fn jit_rb_str_concat( fn jit_rb_ary_empty_p( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -5986,7 +6372,6 @@ fn jit_rb_ary_empty_p( fn jit_rb_ary_length( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -6010,7 +6395,6 @@ fn jit_rb_ary_length( fn jit_rb_ary_push( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -6038,7 +6422,6 @@ fn jit_rb_ary_push( fn jit_rb_hash_empty_p( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -6058,7 +6441,6 @@ fn jit_rb_hash_empty_p( fn jit_obj_respond_to( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -6112,7 +6494,7 @@ fn jit_obj_respond_to( (METHOD_VISI_UNDEF, _) => { // No method, we can return false given respond_to_missing? hasn't been overridden. // In the future, we might want to jit the call to respond_to_missing? - if !assume_method_basic_definition(jit, asm, ocb, recv_class, ID!(respond_to_missing)) { + if !assume_method_basic_definition(jit, asm, recv_class, ID!(respond_to_missing)) { return false; } Qfalse @@ -6134,7 +6516,7 @@ fn jit_obj_respond_to( // Invalidate this block if method lookup changes for the method being queried. This works // both for the case where a method does or does not exist, as for the latter we asked for a // "negative CME" earlier. - jit.assume_method_lookup_stable(asm, ocb, target_cme); + jit.assume_method_lookup_stable(asm, target_cme); if argc == 2 { // pop include_all argument (we only use its type info) @@ -6151,7 +6533,6 @@ fn jit_obj_respond_to( JCC_JNE, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::guard_send_respond_to_mid_mismatch, ); @@ -6164,7 +6545,6 @@ fn jit_obj_respond_to( fn jit_rb_f_block_given_p( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -6179,6 +6559,7 @@ fn jit_rb_f_block_given_p( true } +/// Codegen for `block_given?` and `defined?(yield)` fn gen_block_given( jit: &mut JITState, asm: &mut Assembler, @@ -6188,23 +6569,30 @@ fn gen_block_given( ) { asm_comment!(asm, "block_given?"); - // Same as rb_vm_frame_block_handler - let ep_opnd = gen_get_lep(jit, asm); - let block_handler = asm.load( - Opnd::mem(64, ep_opnd, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL) - ); + // `yield` goes to the block handler stowed in the "local" iseq which is + // the current iseq or a parent. Only the "method" iseq type can be passed a + // block handler. (e.g. `yield` in the top level script is a syntax error.) + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(jit.iseq) }; + if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { + // Same as rb_vm_frame_block_handler + let ep_opnd = gen_get_lep(jit, asm); + let block_handler = asm.load( + Opnd::mem(64, ep_opnd, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL) + ); - // Return `block_handler != VM_BLOCK_HANDLER_NONE` - asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); - let block_given = asm.csel_ne(true_opnd, false_opnd); - asm.mov(out_opnd, block_given); + // Return `block_handler != VM_BLOCK_HANDLER_NONE` + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + let block_given = asm.csel_ne(true_opnd, false_opnd); + asm.mov(out_opnd, block_given); + } else { + asm.mov(out_opnd, false_opnd); + } } // Codegen for rb_class_superclass() fn jit_rb_class_superclass( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, _block: Option<crate::codegen::BlockHandler>, @@ -6215,6 +6603,7 @@ fn jit_rb_class_superclass( fn rb_class_superclass(klass: VALUE) -> VALUE; } + // It may raise "uninitialized class" if !jit_prepare_lazy_frame_call(jit, asm, cme, StackOpnd(0)) { return false; } @@ -6233,14 +6622,13 @@ fn jit_rb_class_superclass( fn jit_rb_case_equal( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, _argc: i32, known_recv_class: Option<VALUE>, ) -> bool { - if !jit.assume_expected_cfunc( asm, ocb, known_recv_class.unwrap(), ID!(eq), rb_obj_equal as _) { + if !jit.assume_expected_cfunc(asm, known_recv_class.unwrap(), ID!(eq), rb_obj_equal as _) { return false; } @@ -6261,7 +6649,6 @@ fn jit_rb_case_equal( fn jit_thread_s_current( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option<BlockHandler>, @@ -6272,7 +6659,7 @@ fn jit_thread_s_current( asm.stack_pop(1); // ec->thread_ptr - let ec_thread_opnd = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR)); + let ec_thread_opnd = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR as i32)); // thread->self let thread_self = Opnd::mem(64, ec_thread_opnd, RUBY_OFFSET_THREAD_SELF); @@ -6282,7 +6669,29 @@ fn jit_thread_s_current( true } -// Check if we know how to codegen for a particular cfunc method +/// Specialization for rb_obj_dup() (Kernel#dup) +fn jit_rb_obj_dup( + _jit: &mut JITState, + asm: &mut Assembler, + _ci: *const rb_callinfo, + _cme: *const rb_callable_method_entry_t, + _block: Option<BlockHandler>, + _argc: i32, + _known_recv_class: Option<VALUE>, +) -> bool { + // Kernel#dup has arity=0, and caller already did argument count check. + let self_type = asm.ctx.get_opnd_type(StackOpnd(0)); + + if self_type.is_imm() { + // Method is no-op when receiver is an immediate value. + true + } else { + false + } +} + +/// Check if we know how to codegen for a particular cfunc method +/// See also: [reg_method_codegen]. fn lookup_cfunc_codegen(def: *const rb_method_definition_t) -> Option<MethodGenFn> { let method_serial = unsafe { get_def_method_serial(def) }; let table = unsafe { METHOD_CODEGEN_TABLE.as_ref().unwrap() }; @@ -6461,14 +6870,6 @@ fn gen_push_frame( asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); - if frame.iseq.is_some() { - // Spill stack temps to let the callee use them (must be done before changing the SP register) - asm.spill_temps(); - - // Saving SP before calculating ep avoids a dependency on a register - // However this must be done after referencing frame.recv, which may be SP-relative - asm.mov(SP, sp); - } let ep = asm.sub(sp, SIZEOF_VALUE.into()); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_EP), ep); } @@ -6476,7 +6877,6 @@ fn gen_push_frame( fn gen_send_cfunc( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, block: Option<BlockHandler>, @@ -6494,11 +6894,11 @@ fn gen_send_cfunc( // If it's a splat and the method expects a Ruby array of arguments if cfunc_argc == -2 && flags & VM_CALL_ARGS_SPLAT != 0 { - gen_counter_incr(asm, Counter::send_cfunc_splat_neg2); + gen_counter_incr(jit, asm, Counter::send_cfunc_splat_neg2); return None; } - exit_if_kwsplat_non_nil(asm, flags, Counter::send_cfunc_kw_splat_non_nil)?; + exit_if_kwsplat_non_nil(jit, asm, flags, Counter::send_cfunc_kw_splat_non_nil)?; let kw_splat = flags & VM_CALL_KW_SPLAT != 0; let kw_arg = unsafe { vm_ci_kwarg(ci) }; @@ -6509,24 +6909,25 @@ fn gen_send_cfunc( }; if kw_arg_num != 0 && flags & VM_CALL_ARGS_SPLAT != 0 { - gen_counter_incr(asm, Counter::send_cfunc_splat_with_kw); + gen_counter_incr(jit, asm, Counter::send_cfunc_splat_with_kw); return None; } if c_method_tracing_currently_enabled(jit) { // Don't JIT if tracing c_call or c_return - gen_counter_incr(asm, Counter::send_cfunc_tracing); + gen_counter_incr(jit, asm, Counter::send_cfunc_tracing); return None; } // Increment total cfunc send count - gen_counter_incr(asm, Counter::num_send_cfunc); + gen_counter_incr(jit, asm, Counter::num_send_cfunc); - // Delegate to codegen for C methods if we have it. + // Delegate to codegen for C methods if we have it and the callsite is simple enough. if kw_arg.is_null() && !kw_splat && flags & VM_CALL_OPT_SEND == 0 && flags & VM_CALL_ARGS_SPLAT == 0 && + flags & VM_CALL_ARGS_BLOCKARG == 0 && (cfunc_argc == -1 || argc == cfunc_argc) { let expected_stack_after = asm.ctx.get_stack_size() as i32 - argc; if let Some(known_cfunc_codegen) = lookup_cfunc_codegen(unsafe { (*cme).def }) { @@ -6535,19 +6936,18 @@ fn gen_send_cfunc( // non-sendish instructions to break this rule as an exception. let cfunc_codegen = if jit.is_sendish() { asm.with_leaf_ccall(|asm| - perf_call!("gen_send_cfunc: ", known_cfunc_codegen(jit, asm, ocb, ci, cme, block, argc, recv_known_class)) + perf_call!("gen_send_cfunc: ", known_cfunc_codegen(jit, asm, ci, cme, block, argc, recv_known_class)) ) } else { - perf_call!("gen_send_cfunc: ", known_cfunc_codegen(jit, asm, ocb, ci, cme, block, argc, recv_known_class)) + perf_call!("gen_send_cfunc: ", known_cfunc_codegen(jit, asm, ci, cme, block, argc, recv_known_class)) }; if cfunc_codegen { assert_eq!(expected_stack_after, asm.ctx.get_stack_size() as i32); - gen_counter_incr(asm, Counter::num_send_cfunc_inline); + gen_counter_incr(jit, asm, Counter::num_send_cfunc_inline); // cfunc codegen generated code. Terminate the block so // there isn't multiple calls in the same block. - jump_to_next_insn(jit, asm, ocb); - return Some(EndBlock); + return jump_to_next_insn(jit, asm); } } } @@ -6569,7 +6969,7 @@ fn gen_send_cfunc( let splat_array_idx = i32::from(kw_splat) + i32::from(block_arg); let comptime_splat_array = jit.peek_at_stack(&asm.ctx, splat_array_idx as isize); if unsafe { rb_yjit_ruby2_keywords_splat_p(comptime_splat_array) } != 0 { - gen_counter_incr(asm, Counter::send_cfunc_splat_varg_ruby2_keywords); + gen_counter_incr(jit, asm, Counter::send_cfunc_splat_varg_ruby2_keywords); return None; } @@ -6598,17 +6998,17 @@ fn gen_send_cfunc( // If the argument count doesn't match if cfunc_argc >= 0 && cfunc_argc != passed_argc && flags & VM_CALL_ARGS_SPLAT == 0 { - gen_counter_incr(asm, Counter::send_cfunc_argc_mismatch); + gen_counter_incr(jit, asm, Counter::send_cfunc_argc_mismatch); return None; } // Don't JIT functions that need C stack arguments for now if cfunc_argc >= 0 && passed_argc + 1 > (C_ARG_OPNDS.len() as i32) { - gen_counter_incr(asm, Counter::send_cfunc_toomany_args); + gen_counter_incr(jit, asm, Counter::send_cfunc_toomany_args); return None; } - let block_arg_type = if block_arg { + let mut block_arg_type = if block_arg { Some(asm.ctx.get_opnd_type(StackOpnd(0))) } else { None @@ -6616,33 +7016,25 @@ fn gen_send_cfunc( match block_arg_type { Some(Type::Nil | Type::BlockParamProxy) => { - // We'll handle this later - } - None => { - // Nothing to do - } - _ => { - gen_counter_incr(asm, Counter::send_cfunc_block_arg); - return None; - } - } - - match block_arg_type { - Some(Type::Nil) => { - // We have a nil block arg, so let's pop it off the args + // We don't need the actual stack value for these asm.stack_pop(1); } - Some(Type::BlockParamProxy) => { - // We don't need the actual stack value + Some(Type::Unknown | Type::UnknownImm) if jit.peek_at_stack(&asm.ctx, 0).nil_p() => { + // The sample blockarg is nil, so speculate that's the case. + asm.cmp(asm.stack_opnd(0), Qnil.into()); + asm.jne(Target::side_exit(Counter::guard_send_cfunc_block_not_nil)); + block_arg_type = Some(Type::Nil); asm.stack_pop(1); } None => { // Nothing to do } _ => { - assert!(false); + gen_counter_incr(jit, asm, Counter::send_cfunc_block_arg); + return None; } } + let block_arg_type = block_arg_type; // drop `mut` // Pop the empty kw_splat hash if kw_splat { @@ -6658,7 +7050,7 @@ fn gen_send_cfunc( let required_args : u32 = (cfunc_argc as u32).saturating_sub(argc as u32 - 1); // + 1 because we pass self if required_args + 1 >= C_ARG_OPNDS.len() as u32 { - gen_counter_incr(asm, Counter::send_cfunc_toomany_args); + gen_counter_incr(jit, asm, Counter::send_cfunc_toomany_args); return None; } @@ -6722,7 +7114,7 @@ fn gen_send_cfunc( cme, recv, sp, - pc: if cfg!(debug_assertions) { + pc: if cfg!(feature = "runtime_checks") { Some(!0) // Poison value. Helps to fail fast. } else { None // Leave PC uninitialized as cfuncs shouldn't read it @@ -6732,7 +7124,7 @@ fn gen_send_cfunc( asm_comment!(asm, "set ec->cfp"); let new_cfp = asm.lea(Opnd::mem(64, CFP, -(RUBY_SIZEOF_CONTROL_FRAME as i32))); - asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), new_cfp); if !kw_arg.is_null() { // Build a hash from all kwargs passed @@ -6815,8 +7207,8 @@ fn gen_send_cfunc( // We also do this after the C call to minimize the impact of spill_temps() on asm.ccall(). if get_option!(gen_stats) { // Assemble the method name string - let mid = unsafe { vm_ci_mid(ci) }; - let name_str = get_method_name(recv_known_class, mid); + let mid = unsafe { rb_get_def_original_id((*cme).def) }; + let name_str = get_method_name(Some(unsafe { (*cme).owner }), mid); // Get an index for this cfunc name let cfunc_idx = get_cfunc_idx(&name_str); @@ -6828,7 +7220,7 @@ fn gen_send_cfunc( // Pop the stack frame (ec->cfp++) // Instead of recalculating, we can reuse the previous CFP, which is stored in a callee-saved // register - let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP); + let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32); asm.store(ec_cfp_opnd, CFP); // cfunc calls may corrupt types @@ -6839,8 +7231,7 @@ fn gen_send_cfunc( // Jump (fall through) to the call continuation block // We do this to end the current block after the call - jump_to_next_insn(jit, asm, ocb); - Some(EndBlock) + jump_to_next_insn(jit, asm) } // Generate RARRAY_LEN. For array_opnd, use Opnd::Reg to reduce memory access, @@ -6987,7 +7378,6 @@ fn push_splat_args(required_args: u32, asm: &mut Assembler) { fn gen_send_bmethod( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, block: Option<BlockHandler>, @@ -6996,7 +7386,7 @@ fn gen_send_bmethod( ) -> Option<CodegenStatus> { let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; - let proc = unsafe { rb_yjit_get_proc_ptr(procv) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; let proc_block = unsafe { &(*proc).block }; if proc_block.type_ != block_type_iseq { @@ -7006,22 +7396,23 @@ fn gen_send_bmethod( let capture = unsafe { proc_block.as_.captured.as_ref() }; let iseq = unsafe { *capture.code.iseq.as_ref() }; - // Optimize for single ractor mode and avoid runtime check for - // "defined with an un-shareable Proc in a different Ractor" - if !assume_single_ractor_mode(jit, asm, ocb) { - gen_counter_incr(asm, Counter::send_bmethod_ractor); - return None; + if !procv.shareable_p() { + let ractor_serial = unsafe { rb_yjit_cme_ractor_serial(cme) }; + asm_comment!(asm, "guard current ractor == {}", ractor_serial); + let current_ractor_serial = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_RACTOR_ID as i32)); + asm.cmp(current_ractor_serial, ractor_serial.into()); + asm.jne(Target::side_exit(Counter::send_bmethod_ractor)); } // Passing a block to a block needs logic different from passing // a block to a method and sometimes requires allocation. Bail for now. if block.is_some() { - gen_counter_incr(asm, Counter::send_bmethod_block_arg); + gen_counter_incr(jit, asm, Counter::send_bmethod_block_arg); return None; } let frame_type = VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA; - perf_call! { gen_send_iseq(jit, asm, ocb, iseq, ci, frame_type, Some(capture.ep), cme, block, flags, argc, None) } + perf_call! { gen_send_iseq(jit, asm, iseq, ci, frame_type, Some(capture.ep), cme, block, flags, argc, None) } } /// The kind of a value an ISEQ returns @@ -7031,13 +7422,16 @@ enum IseqReturn { Receiver, } -extern { +extern "C" { fn rb_simple_iseq_p(iseq: IseqPtr) -> bool; + fn rb_iseq_only_kwparam_p(iseq: IseqPtr) -> bool; } /// Return the ISEQ's return value if it consists of one simple instruction and leave. -fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<Opnd>, ci_flags: u32) -> Option<IseqReturn> { +fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<Opnd>, block: Option<BlockHandler>, ci_flags: u32) -> Option<IseqReturn> { // Expect only two instructions and one possible operand + // NOTE: If an ISEQ has an optional keyword parameter with a default value that requires + // computation, the ISEQ will always have more than two instructions and won't be inlined. let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; if !(2..=3).contains(&iseq_size) { return None; @@ -7053,15 +7447,44 @@ fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<Opnd>, ci_flags: u } match first_insn { YARVINSN_getlocal_WC_0 => { - // Only accept simple positional only cases for both the caller and the callee. + // Accept only cases where only positional arguments are used by both the callee and the caller. + // Keyword arguments may be specified by the callee or the caller but not used. // Reject block ISEQs to avoid autosplat and other block parameter complications. - if captured_opnd.is_none() && unsafe { rb_simple_iseq_p(iseq) } && ci_flags & VM_CALL_ARGS_SIMPLE != 0 { - let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32(); - let local_idx = ep_offset_to_local_idx(iseq, ep_offset); - Some(IseqReturn::LocalVariable(local_idx)) - } else { - None + if captured_opnd.is_some() + // Reject if block ISEQ is present + || block.is_some() + // Equivalent to `VM_CALL_ARGS_SIMPLE - VM_CALL_KWARG - has_block_iseq` + || ci_flags & ( + VM_CALL_ARGS_SPLAT + | VM_CALL_KW_SPLAT + | VM_CALL_ARGS_BLOCKARG + | VM_CALL_FORWARDING + ) != 0 + { + return None; + } + + let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32(); + let local_idx = ep_offset_to_local_idx(iseq, ep_offset); + + // Only inline getlocal on a parameter. DCE in the IESQ builder can + // make a two-instruction ISEQ that does not return a parameter. + if local_idx >= unsafe { get_iseq_body_param_size(iseq) } { + return None; } + + if unsafe { rb_simple_iseq_p(iseq) } { + return Some(IseqReturn::LocalVariable(local_idx)); + } else if unsafe { rb_iseq_only_kwparam_p(iseq) } { + // Inline if only positional parameters are used + if let Ok(i) = i32::try_from(local_idx) { + if i < unsafe { rb_get_iseq_body_param_lead_num(iseq) } { + return Some(IseqReturn::LocalVariable(local_idx)); + } + } + } + + return None; } YARVINSN_putnil => Some(IseqReturn::Value(Qnil)), YARVINSN_putobject => Some(IseqReturn::Value(unsafe { *rb_iseq_pc_at_idx(iseq, 1) })), @@ -7076,7 +7499,6 @@ fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<Opnd>, ci_flags: u fn gen_send_iseq( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, iseq: *const rb_iseq_t, ci: *const rb_callinfo, frame_type: u32, @@ -7107,8 +7529,19 @@ fn gen_send_iseq( let iseq_has_rest = unsafe { get_iseq_flags_has_rest(iseq) }; let iseq_has_block_param = unsafe { get_iseq_flags_has_block(iseq) }; let arg_setup_block = captured_opnd.is_some(); // arg_setup_type: arg_setup_block (invokeblock) - let kw_splat = flags & VM_CALL_KW_SPLAT != 0; - let splat_call = flags & VM_CALL_ARGS_SPLAT != 0; + + // Is this iseq tagged as "forwardable"? Iseqs that take `...` as a + // parameter are tagged as forwardable (e.g. `def foo(...); end`) + let forwarding = unsafe { rb_get_iseq_flags_forwardable(iseq) }; + + // If a "forwardable" iseq has been called with a splat, then we _do not_ + // want to expand the splat to the stack. So we'll only consider this + // a splat call if the callee iseq is not forwardable. For example, + // we do not want to handle the following code: + // + // `def foo(...); end; foo(*blah)` + let splat_call = (flags & VM_CALL_ARGS_SPLAT != 0) && !forwarding; + let kw_splat = (flags & VM_CALL_KW_SPLAT != 0) && !forwarding; // For computing offsets to callee locals let num_params = unsafe { get_iseq_body_param_size(iseq) as i32 }; @@ -7143,23 +7576,29 @@ fn gen_send_iseq( let splat_pos = i32::from(block_arg) + i32::from(kw_splat) + kw_arg_num; exit_if_stack_too_large(iseq)?; - exit_if_tail_call(asm, ci)?; - exit_if_has_post(asm, iseq)?; - exit_if_kwsplat_non_nil(asm, flags, Counter::send_iseq_kw_splat_non_nil)?; - exit_if_has_rest_and_captured(asm, iseq_has_rest, captured_opnd)?; - exit_if_has_kwrest_and_captured(asm, has_kwrest, captured_opnd)?; - exit_if_has_rest_and_supplying_kws(asm, iseq_has_rest, supplying_kws)?; - exit_if_supplying_kw_and_has_no_kw(asm, supplying_kws, doing_kw_call)?; - exit_if_supplying_kws_and_accept_no_kwargs(asm, supplying_kws, iseq)?; - exit_if_doing_kw_and_splat(asm, doing_kw_call, flags)?; - exit_if_wrong_number_arguments(asm, arg_setup_block, opts_filled, flags, opt_num, iseq_has_rest)?; - exit_if_doing_kw_and_opts_missing(asm, doing_kw_call, opts_missing)?; - exit_if_has_rest_and_optional_and_block(asm, iseq_has_rest, opt_num, iseq, block_arg)?; + exit_if_tail_call(jit, asm, ci)?; + exit_if_has_post(jit, asm, iseq)?; + exit_if_kwsplat_non_nil(jit, asm, flags, Counter::send_iseq_kw_splat_non_nil)?; + exit_if_has_rest_and_captured(jit, asm, iseq_has_rest, captured_opnd)?; + exit_if_has_kwrest_and_captured(jit, asm, has_kwrest, captured_opnd)?; + exit_if_has_rest_and_supplying_kws(jit, asm, iseq_has_rest, supplying_kws)?; + exit_if_supplying_kw_and_has_no_kw(jit, asm, supplying_kws, doing_kw_call)?; + exit_if_supplying_kws_and_accept_no_kwargs(jit, asm, supplying_kws, iseq)?; + exit_if_doing_kw_and_splat(jit, asm, doing_kw_call, flags)?; + if !forwarding { + exit_if_wrong_number_arguments(jit, asm, arg_setup_block, opts_filled, flags, opt_num, iseq_has_rest)?; + } + exit_if_doing_kw_and_opts_missing(jit, asm, doing_kw_call, opts_missing)?; + exit_if_has_rest_and_optional_and_block(jit, asm, iseq_has_rest, opt_num, iseq, block_arg)?; + if forwarding && flags & VM_CALL_OPT_SEND != 0 { + gen_counter_incr(jit, asm, Counter::send_iseq_send_forwarding); + return None; + } let block_arg_type = exit_if_unsupported_block_arg_type(jit, asm, block_arg)?; // Bail if we can't drop extra arguments for a yield by just popping them if supplying_kws && arg_setup_block && argc > (kw_arg_num + required_num + opt_num) { - gen_counter_incr(asm, Counter::send_iseq_complex_discard_extras); + gen_counter_incr(jit, asm, Counter::send_iseq_complex_discard_extras); return None; } @@ -7171,7 +7610,7 @@ fn gen_send_iseq( // In this case (param.flags.has_block && local_iseq != iseq), // the block argument is setup as a local variable and requires // materialization (allocation). Bail. - gen_counter_incr(asm, Counter::send_iseq_materialized_block); + gen_counter_incr(jit, asm, Counter::send_iseq_materialized_block); return None; } } @@ -7179,7 +7618,7 @@ fn gen_send_iseq( // Check that required keyword arguments are supplied and find any extras // that should go into the keyword rest parameter (**kw_rest). if doing_kw_call { - gen_iseq_kw_call_checks(asm, iseq, kw_arg, has_kwrest, kw_arg_num)?; + gen_iseq_kw_call_checks(jit, asm, iseq, kw_arg, has_kwrest, kw_arg_num)?; } let splat_array_length = if splat_call { @@ -7187,10 +7626,10 @@ fn gen_send_iseq( let array_length = if array == Qnil { 0 } else if unsafe { !RB_TYPE_P(array, RUBY_T_ARRAY) } { - gen_counter_incr(asm, Counter::send_iseq_splat_not_array); + gen_counter_incr(jit, asm, Counter::send_iseq_splat_not_array); return None; } else { - unsafe { rb_yjit_array_len(array) as u32} + unsafe { rb_jit_array_len(array) as u32} }; // Arity check accounting for size of the splat. When callee has rest parameters, we insert @@ -7198,7 +7637,7 @@ fn gen_send_iseq( if !iseq_has_rest { let supplying = argc - 1 - i32::from(kw_splat) + array_length as i32; if (required_num..=required_num + opt_num).contains(&supplying) == false { - gen_counter_incr(asm, Counter::send_iseq_splat_arity_error); + gen_counter_incr(jit, asm, Counter::send_iseq_splat_arity_error); return None; } } @@ -7237,14 +7676,14 @@ fn gen_send_iseq( // If block_arg0_splat, we still need side exits after splat, but // the splat modifies the stack which breaks side exits. So bail out. if splat_call { - gen_counter_incr(asm, Counter::invokeblock_iseq_arg0_args_splat); + gen_counter_incr(jit, asm, Counter::invokeblock_iseq_arg0_args_splat); return None; } // The block_arg0_splat implementation cannot deal with optional parameters. // This is a setup_parameters_complex() situation and interacts with the // starting position of the callee. if opt_num > 1 { - gen_counter_incr(asm, Counter::invokeblock_iseq_arg0_optional); + gen_counter_incr(jit, asm, Counter::invokeblock_iseq_arg0_optional); return None; } } @@ -7278,10 +7717,10 @@ fn gen_send_iseq( } // Increment total ISEQ send count - gen_counter_incr(asm, Counter::num_send_iseq); + gen_counter_incr(jit, asm, Counter::num_send_iseq); // Shortcut for special `Primitive.attr! :leaf` builtins - let builtin_attrs = unsafe { rb_yjit_iseq_builtin_attrs(iseq) }; + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; let builtin_func_raw = unsafe { rb_yjit_builtin_function(iseq) }; let builtin_func = if builtin_func_raw.is_null() { None } else { Some(builtin_func_raw) }; let opt_send_call = flags & VM_CALL_OPT_SEND != 0; // .send call is not currently supported for builtins @@ -7295,7 +7734,7 @@ fn gen_send_iseq( // adding one requires interpreter changes to support. if block_arg_type.is_some() { if iseq_has_block_param { - gen_counter_incr(asm, Counter::send_iseq_leaf_builtin_block_arg_block_param); + gen_counter_incr(jit, asm, Counter::send_iseq_leaf_builtin_block_arg_block_param); return None; } asm.stack_pop(1); @@ -7312,7 +7751,7 @@ fn gen_send_iseq( } asm_comment!(asm, "inlined leaf builtin"); - gen_counter_incr(asm, Counter::num_send_iseq_leaf); + gen_counter_incr(jit, asm, Counter::num_send_iseq_leaf); // The callee may allocate, e.g. Integer#abs on a Bignum. // Save SP for GC, save PC for allocation tracing, and prepare @@ -7338,15 +7777,14 @@ fn gen_send_iseq( // Seems like a safe assumption. // Let guard chains share the same successor - jump_to_next_insn(jit, asm, ocb); - return Some(EndBlock); + return jump_to_next_insn(jit, asm); } } // Inline simple ISEQs whose return value is known at compile time - if let (Some(value), None, false) = (iseq_get_return_value(iseq, captured_opnd, flags), block_arg_type, opt_send_call) { + if let (Some(value), None, false) = (iseq_get_return_value(iseq, captured_opnd, block, flags), block_arg_type, opt_send_call) { asm_comment!(asm, "inlined simple ISEQ"); - gen_counter_incr(asm, Counter::num_send_iseq_inline); + gen_counter_incr(jit, asm, Counter::num_send_iseq_inline); match value { IseqReturn::LocalVariable(local_idx) => { @@ -7377,8 +7815,7 @@ fn gen_send_iseq( } // Let guard chains share the same successor - jump_to_next_insn(jit, asm, ocb); - return Some(EndBlock); + return jump_to_next_insn(jit, asm); } // Stack overflow check @@ -7444,16 +7881,25 @@ fn gen_send_iseq( JCC_JE, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::guard_send_block_arg_type, ); - let callee_ep = -argc + num_locals + VM_ENV_DATA_SIZE as i32 - 1; + // If this is a forwardable iseq, adjust the stack size accordingly + let callee_ep = if forwarding { + -1 + num_locals + VM_ENV_DATA_SIZE as i32 + } else { + -argc + num_locals + VM_ENV_DATA_SIZE as i32 - 1 + }; let callee_specval = callee_ep + VM_ENV_DATA_INDEX_SPECVAL; if callee_specval < 0 { // Can't write to sp[-n] since that's where the arguments are - gen_counter_incr(asm, Counter::send_iseq_clobbering_block_arg); + gen_counter_incr(jit, asm, Counter::send_iseq_clobbering_block_arg); + return None; + } + if iseq_has_rest || has_kwrest { + // The proc would be stored above the current stack top, where GC can't see it + gen_counter_incr(jit, asm, Counter::send_iseq_block_arg_gc_unsafe); return None; } let proc = asm.stack_pop(1); // Pop first, as argc doesn't account for the block arg @@ -7479,7 +7925,7 @@ fn gen_send_iseq( // an array that has the same length. We will insert guards. argc = argc - 1 + array_length as i32; if argc + asm.ctx.get_stack_size() as i32 > MAX_SPLAT_LENGTH { - gen_counter_incr(asm, Counter::send_splat_too_long); + gen_counter_incr(jit, asm, Counter::send_splat_too_long); return None; } push_splat_args(array_length, asm); @@ -7600,20 +8046,20 @@ fn gen_send_iseq( }; // Store rest param to memory to avoid register shuffle as // we won't be reading it for the remainder of the block. - asm.ctx.dealloc_temp_reg(rest_param.stack_idx()); + asm.ctx.dealloc_reg(rest_param.reg_opnd()); asm.store(rest_param, rest_param_array); } // Pop surplus positional arguments when yielding if arg_setup_block { - let extras = argc - required_num - opt_num; + let extras = argc - required_num - opt_num - kw_arg_num; if extras > 0 { // Checked earlier. If there are keyword args, then // the positional arguments are not at the stack top. assert_eq!(0, kw_arg_num); asm.stack_pop(extras as usize); - argc = required_num + opt_num; + argc = required_num + opt_num + kw_arg_num; } } @@ -7663,33 +8109,48 @@ fn gen_send_iseq( } } - // Nil-initialize missing optional parameters - nil_fill( - "nil-initialize missing optionals", - { - let begin = -argc + required_num + opts_filled; - let end = -argc + required_num + opt_num; + if !forwarding { + // Nil-initialize missing optional parameters + nil_fill( + "nil-initialize missing optionals", + { + let begin = -argc + required_num + opts_filled; + let end = -argc + required_num + opt_num; - begin..end - }, - asm - ); - // Nil-initialize the block parameter. It's the last parameter local - if iseq_has_block_param { - let block_param = asm.ctx.sp_opnd(-argc + num_params - 1); - asm.store(block_param, Qnil.into()); + begin..end + }, + asm + ); + // Nil-initialize the block parameter. It's the last parameter local + if iseq_has_block_param { + let block_param = asm.ctx.sp_opnd(-argc + num_params - 1); + asm.store(block_param, Qnil.into()); + } + // Nil-initialize non-parameter locals + nil_fill( + "nil-initialize locals", + { + let begin = -argc + num_params; + let end = -argc + num_locals; + + begin..end + }, + asm + ); } - // Nil-initialize non-parameter locals - nil_fill( - "nil-initialize locals", - { - let begin = -argc + num_params; - let end = -argc + num_locals; - begin..end - }, - asm - ); + if forwarding { + assert_eq!(1, num_params); + // Write the CI in to the stack and ensure that it actually gets + // flushed to memory + asm_comment!(asm, "put call info for forwarding"); + let ci_opnd = asm.stack_opnd(-1); + asm.ctx.dealloc_reg(ci_opnd.reg_opnd()); + asm.mov(ci_opnd, VALUE(ci as usize).into()); + + // Nil-initialize other locals which are above the CI + nil_fill("nil-initialize locals", 1..num_locals, asm); + } // Points to the receiver operand on the stack unless a captured environment is used let recv = match captured_opnd { @@ -7708,7 +8169,13 @@ fn gen_send_iseq( jit_save_pc(jit, asm); // Adjust the callee's stack pointer - let callee_sp = asm.lea(asm.ctx.sp_opnd(-argc + num_locals + VM_ENV_DATA_SIZE as i32)); + let callee_sp = if forwarding { + let offs = num_locals + VM_ENV_DATA_SIZE as i32; + asm.lea(asm.ctx.sp_opnd(offs)) + } else { + let offs = -argc + num_locals + VM_ENV_DATA_SIZE as i32; + asm.lea(asm.ctx.sp_opnd(offs)) + }; let specval = if let Some(prev_ep) = prev_ep { // We've already side-exited if the callee expects a block, so we @@ -7736,29 +8203,13 @@ fn gen_send_iseq( pc: None, // We are calling into jitted code, which will set the PC as necessary })); - // Log the name of the method we're calling to. We intentionally don't do this for inlined ISEQs. - // We also do this after gen_push_frame() to minimize the impact of spill_temps() on asm.ccall(). - if get_option!(gen_stats) { - // Assemble the ISEQ name string - let name_str = get_iseq_name(iseq); - - // Get an index for this ISEQ name - let iseq_idx = get_iseq_idx(&name_str); - - // Increment the counter for this cfunc - asm.ccall(incr_iseq_counter as *const u8, vec![iseq_idx.into()]); - } - // No need to set cfp->pc since the callee sets it whenever calling into routines // that could look at it through jit_save_pc(). // mov(cb, REG0, const_ptr_opnd(start_pc)); // mov(cb, member_opnd(REG_CFP, rb_control_frame_t, pc), REG0); - // Stub so we can return to JITted code - let return_block = BlockId { - iseq: jit.iseq, - idx: jit.next_insn_idx(), - }; + // Create a blockid for the callee + let callee_blockid = BlockId { iseq, idx: start_pc_offset }; // Create a context for the callee let mut callee_ctx = Context::default(); @@ -7776,6 +8227,13 @@ fn gen_send_iseq( callee_ctx.set_local_type(arg_idx.try_into().unwrap(), arg_type); } + // If we're in a forwarding callee, there will be one unknown type + // written in to the local table (the caller's CI object) + if forwarding { + callee_ctx.set_local_type(0, Type::Unknown) + } + + // Set the receiver type in the callee's context let recv_type = if captured_self { Type::Unknown // we don't track the type information of captured->self for now } else { @@ -7783,23 +8241,113 @@ fn gen_send_iseq( }; callee_ctx.upgrade_opnd_type(SelfOpnd, recv_type); + // Spill or preserve argument registers + if forwarding { + // When forwarding, the callee's local table has only a callinfo, + // so we can't map the actual arguments to the callee's locals. + asm.spill_regs(); + } else { + // Discover stack temp registers that can be used as the callee's locals + let mapped_temps = asm.map_temp_regs_to_args(&mut callee_ctx, argc); + + // Spill stack temps and locals that are not used by the callee. + // This must be done before changing the SP register. + asm.spill_regs_except(&mapped_temps); + + // If the callee block has been compiled before, spill/move registers to reuse the existing block + // for minimizing the number of blocks we need to compile. + if let Some(existing_reg_mapping) = find_most_compatible_reg_mapping(callee_blockid, &callee_ctx) { + asm_comment!(asm, "reuse maps: {:?} -> {:?}", callee_ctx.get_reg_mapping(), existing_reg_mapping); + + // Spill the registers that are not used in the existing block. + // When the same ISEQ is compiled as an entry block, it starts with no registers allocated. + for ®_opnd in callee_ctx.get_reg_mapping().get_reg_opnds().iter() { + if existing_reg_mapping.get_reg(reg_opnd).is_none() { + match reg_opnd { + RegOpnd::Local(local_idx) => { + let spilled_temp = asm.stack_opnd(argc - local_idx as i32 - 1); + asm.spill_reg(spilled_temp); + callee_ctx.dealloc_reg(reg_opnd); + } + RegOpnd::Stack(_) => unreachable!("callee {:?} should have been spilled", reg_opnd), + } + } + } + assert!(callee_ctx.get_reg_mapping().get_reg_opnds().len() <= existing_reg_mapping.get_reg_opnds().len()); + + // Load the registers that are spilled in this block but used in the existing block. + // When there are multiple callsites, some registers spilled in this block may be used at other callsites. + for ®_opnd in existing_reg_mapping.get_reg_opnds().iter() { + if callee_ctx.get_reg_mapping().get_reg(reg_opnd).is_none() { + match reg_opnd { + RegOpnd::Local(local_idx) => { + callee_ctx.alloc_reg(reg_opnd); + let loaded_reg = TEMP_REGS[callee_ctx.get_reg_mapping().get_reg(reg_opnd).unwrap()]; + let loaded_temp = asm.stack_opnd(argc - local_idx as i32 - 1); + asm.load_into(Opnd::Reg(loaded_reg), loaded_temp); + } + RegOpnd::Stack(_) => unreachable!("find_most_compatible_reg_mapping should not leave {:?}", reg_opnd), + } + } + } + assert_eq!(callee_ctx.get_reg_mapping().get_reg_opnds().len(), existing_reg_mapping.get_reg_opnds().len()); + + // Shuffle registers to make the register mappings compatible + let mut moves = vec![]; + for ®_opnd in callee_ctx.get_reg_mapping().get_reg_opnds().iter() { + let old_reg = TEMP_REGS[callee_ctx.get_reg_mapping().get_reg(reg_opnd).unwrap()]; + let new_reg = TEMP_REGS[existing_reg_mapping.get_reg(reg_opnd).unwrap()]; + moves.push((new_reg, Opnd::Reg(old_reg))); + } + for (reg, opnd) in Assembler::reorder_reg_moves(&moves) { + asm.load_into(Opnd::Reg(reg), opnd); + } + callee_ctx.set_reg_mapping(existing_reg_mapping); + } + } + + // Update SP register for the callee. This must be done after referencing frame.recv, + // which may be SP-relative. + asm.mov(SP, callee_sp); + + // Log the name of the method we're calling to. We intentionally don't do this for inlined ISEQs. + // We also do this after spill_regs() to avoid doubly spilling the same thing on asm.ccall(). + if get_option!(gen_stats) { + // Protect caller-saved registers in case they're used for arguments + let mapping = asm.cpush_all(); + + // Assemble the ISEQ name string + let name_str = get_iseq_name(iseq); + + // Get an index for this ISEQ name + let iseq_idx = get_iseq_idx(&name_str); + + // Increment the counter for this cfunc + asm.ccall(incr_iseq_counter as *const u8, vec![iseq_idx.into()]); + asm.cpop_all(mapping); + } + // The callee might change locals through Kernel#binding and other means. asm.clear_local_types(); // Pop arguments and receiver in return context and // mark it as a continuation of gen_leave() - let mut return_asm = Assembler::new(); + let mut return_asm = Assembler::new(jit.num_locals()); return_asm.ctx = asm.ctx; return_asm.stack_pop(sp_offset.try_into().unwrap()); return_asm.ctx.set_sp_offset(0); // We set SP on the caller's frame above return_asm.ctx.reset_chain_depth_and_defer(); return_asm.ctx.set_as_return_landing(); + // Stub so we can return to JITted code + let return_block = BlockId { + iseq: jit.iseq, + idx: jit.next_insn_idx(), + }; + // Write the JIT return address on the callee frame - gen_branch( - jit, + jit.gen_branch( asm, - ocb, return_block, &return_asm.ctx, None, @@ -7811,16 +8359,13 @@ fn gen_send_iseq( asm_comment!(asm, "switch to new CFP"); let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); asm.mov(CFP, new_cfp); - asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); // Directly jump to the entry point of the callee gen_direct_jump( jit, &callee_ctx, - BlockId { - iseq: iseq, - idx: start_pc_offset, - }, + callee_blockid, asm, ); @@ -7829,6 +8374,7 @@ fn gen_send_iseq( // Check if we can handle a keyword call fn gen_iseq_kw_call_checks( + jit: &JITState, asm: &mut Assembler, iseq: *const rb_iseq_t, kw_arg: *const rb_callinfo_kwarg, @@ -7847,7 +8393,7 @@ fn gen_iseq_kw_call_checks( // We have so many keywords that (1 << num) encoded as a FIXNUM // (which shifts it left one more) no longer fits inside a 32-bit // immediate. Similarly, we use a u64 in case of keyword rest parameter. - gen_counter_incr(asm, Counter::send_iseq_too_many_kwargs); + gen_counter_incr(jit, asm, Counter::send_iseq_too_many_kwargs); return None; } @@ -7888,7 +8434,7 @@ fn gen_iseq_kw_call_checks( // If the keyword was never found, then we know we have a // mismatch in the names of the keyword arguments, so we need to // bail. - gen_counter_incr(asm, Counter::send_iseq_kwargs_mismatch); + gen_counter_incr(jit, asm, Counter::send_iseq_kwargs_mismatch); return None; } Some((callee_idx, _)) if callee_idx < keyword_required_num => { @@ -7901,7 +8447,7 @@ fn gen_iseq_kw_call_checks( } assert!(required_kwargs_filled <= keyword_required_num); if required_kwargs_filled != keyword_required_num { - gen_counter_incr(asm, Counter::send_iseq_kwargs_mismatch); + gen_counter_incr(jit, asm, Counter::send_iseq_kwargs_mismatch); return None; } @@ -8042,10 +8588,10 @@ fn gen_iseq_kw_call( kwargs_order[kwrest_idx] = 0; } // Put kwrest straight into memory, since we might pop it later - asm.ctx.dealloc_temp_reg(stack_kwrest.stack_idx()); + asm.ctx.dealloc_reg(stack_kwrest.reg_opnd()); asm.mov(stack_kwrest, kwrest); if stack_kwrest_idx >= 0 { - asm.ctx.set_opnd_mapping(stack_kwrest.into(), TempMapping::map_to_stack(kwrest_type)); + asm.ctx.set_opnd_mapping(stack_kwrest.into(), TempMapping::MapToStack(kwrest_type)); } Some(kwrest_type) @@ -8115,7 +8661,7 @@ fn gen_iseq_kw_call( let default_param = asm.stack_opnd(kwargs_stack_base - kwarg_idx as i32); let param_type = Type::from(default_value); asm.mov(default_param, default_value.into()); - asm.ctx.set_opnd_mapping(default_param.into(), TempMapping::map_to_stack(param_type)); + asm.ctx.set_opnd_mapping(default_param.into(), TempMapping::MapToStack(param_type)); } } @@ -8140,7 +8686,7 @@ fn gen_iseq_kw_call( if let Some(kwrest_type) = kwrest_type { let kwrest = asm.stack_push(kwrest_type); // We put the kwrest parameter in memory earlier - asm.ctx.dealloc_temp_reg(kwrest.stack_idx()); + asm.ctx.dealloc_reg(kwrest.reg_opnd()); argc += 1; } @@ -8153,49 +8699,50 @@ fn gen_iseq_kw_call( /// short-circuit using the ? operator if we return None. /// It would be great if rust let you implement ? for your /// own types, but as of right now they don't. -fn exit_if(asm: &mut Assembler, pred: bool, counter: Counter) -> Option<()> { +fn exit_if(jit: &JITState, asm: &mut Assembler, pred: bool, counter: Counter) -> Option<()> { if pred { - gen_counter_incr(asm, counter); + gen_counter_incr(jit, asm, counter); return None } Some(()) } #[must_use] -fn exit_if_tail_call(asm: &mut Assembler, ci: *const rb_callinfo) -> Option<()> { - exit_if(asm, unsafe { vm_ci_flag(ci) } & VM_CALL_TAILCALL != 0, Counter::send_iseq_tailcall) +fn exit_if_tail_call(jit: &JITState, asm: &mut Assembler, ci: *const rb_callinfo) -> Option<()> { + exit_if(jit, asm, unsafe { vm_ci_flag(ci) } & VM_CALL_TAILCALL != 0, Counter::send_iseq_tailcall) } #[must_use] -fn exit_if_has_post(asm: &mut Assembler, iseq: *const rb_iseq_t) -> Option<()> { - exit_if(asm, unsafe { get_iseq_flags_has_post(iseq) }, Counter::send_iseq_has_post) +fn exit_if_has_post(jit: &JITState, asm: &mut Assembler, iseq: *const rb_iseq_t) -> Option<()> { + exit_if(jit, asm, unsafe { get_iseq_flags_has_post(iseq) }, Counter::send_iseq_has_post) } #[must_use] -fn exit_if_kwsplat_non_nil(asm: &mut Assembler, flags: u32, counter: Counter) -> Option<()> { +fn exit_if_kwsplat_non_nil(jit: &JITState, asm: &mut Assembler, flags: u32, counter: Counter) -> Option<()> { let kw_splat = flags & VM_CALL_KW_SPLAT != 0; let kw_splat_stack = StackOpnd((flags & VM_CALL_ARGS_BLOCKARG != 0).into()); - exit_if(asm, kw_splat && asm.ctx.get_opnd_type(kw_splat_stack) != Type::Nil, counter) + exit_if(jit, asm, kw_splat && asm.ctx.get_opnd_type(kw_splat_stack) != Type::Nil, counter) } #[must_use] -fn exit_if_has_rest_and_captured(asm: &mut Assembler, iseq_has_rest: bool, captured_opnd: Option<Opnd>) -> Option<()> { - exit_if(asm, iseq_has_rest && captured_opnd.is_some(), Counter::send_iseq_has_rest_and_captured) +fn exit_if_has_rest_and_captured(jit: &JITState, asm: &mut Assembler, iseq_has_rest: bool, captured_opnd: Option<Opnd>) -> Option<()> { + exit_if(jit, asm, iseq_has_rest && captured_opnd.is_some(), Counter::send_iseq_has_rest_and_captured) } #[must_use] -fn exit_if_has_kwrest_and_captured(asm: &mut Assembler, iseq_has_kwrest: bool, captured_opnd: Option<Opnd>) -> Option<()> { +fn exit_if_has_kwrest_and_captured(jit: &JITState, asm: &mut Assembler, iseq_has_kwrest: bool, captured_opnd: Option<Opnd>) -> Option<()> { // We need to call a C function to allocate the kwrest hash, but also need to hold the captred // block across the call, which we can't do. - exit_if(asm, iseq_has_kwrest && captured_opnd.is_some(), Counter::send_iseq_has_kwrest_and_captured) + exit_if(jit, asm, iseq_has_kwrest && captured_opnd.is_some(), Counter::send_iseq_has_kwrest_and_captured) } #[must_use] -fn exit_if_has_rest_and_supplying_kws(asm: &mut Assembler, iseq_has_rest: bool, supplying_kws: bool) -> Option<()> { +fn exit_if_has_rest_and_supplying_kws(jit: &JITState, asm: &mut Assembler, iseq_has_rest: bool, supplying_kws: bool) -> Option<()> { // There can be a gap between the rest parameter array and the supplied keywords, or // no space to put the rest array (e.g. `def foo(*arr, k:) = arr; foo(k: 1)` 1 is // sitting where the rest array should be). exit_if( + jit, asm, iseq_has_rest && supplying_kws, Counter::send_iseq_has_rest_and_kw_supplied, @@ -8203,10 +8750,11 @@ fn exit_if_has_rest_and_supplying_kws(asm: &mut Assembler, iseq_has_rest: bool, } #[must_use] -fn exit_if_supplying_kw_and_has_no_kw(asm: &mut Assembler, supplying_kws: bool, callee_kws: bool) -> Option<()> { +fn exit_if_supplying_kw_and_has_no_kw(jit: &JITState, asm: &mut Assembler, supplying_kws: bool, callee_kws: bool) -> Option<()> { // Passing keyword arguments to a callee means allocating a hash and treating // that as a positional argument. Bail for now. exit_if( + jit, asm, supplying_kws && !callee_kws, Counter::send_iseq_has_no_kw, @@ -8214,10 +8762,11 @@ fn exit_if_supplying_kw_and_has_no_kw(asm: &mut Assembler, supplying_kws: bool, } #[must_use] -fn exit_if_supplying_kws_and_accept_no_kwargs(asm: &mut Assembler, supplying_kws: bool, iseq: *const rb_iseq_t) -> Option<()> { +fn exit_if_supplying_kws_and_accept_no_kwargs(jit: &JITState, asm: &mut Assembler, supplying_kws: bool, iseq: *const rb_iseq_t) -> Option<()> { // If we have a method accepting no kwargs (**nil), exit if we have passed // it any kwargs. exit_if( + jit, asm, supplying_kws && unsafe { get_iseq_flags_accepts_no_kwarg(iseq) }, Counter::send_iseq_accepts_no_kwarg @@ -8225,12 +8774,13 @@ fn exit_if_supplying_kws_and_accept_no_kwargs(asm: &mut Assembler, supplying_kws } #[must_use] -fn exit_if_doing_kw_and_splat(asm: &mut Assembler, doing_kw_call: bool, flags: u32) -> Option<()> { - exit_if(asm, doing_kw_call && flags & VM_CALL_ARGS_SPLAT != 0, Counter::send_iseq_splat_with_kw) +fn exit_if_doing_kw_and_splat(jit: &JITState, asm: &mut Assembler, doing_kw_call: bool, flags: u32) -> Option<()> { + exit_if(jit, asm, doing_kw_call && flags & VM_CALL_ARGS_SPLAT != 0, Counter::send_iseq_splat_with_kw) } #[must_use] fn exit_if_wrong_number_arguments( + jit: &JITState, asm: &mut Assembler, args_setup_block: bool, opts_filled: i32, @@ -8243,20 +8793,21 @@ fn exit_if_wrong_number_arguments( // Too many arguments and no sink that take them let too_many = opts_filled > opt_num && !(iseq_has_rest || args_setup_block); - exit_if(asm, too_few || too_many, Counter::send_iseq_arity_error) + exit_if(jit, asm, too_few || too_many, Counter::send_iseq_arity_error) } #[must_use] -fn exit_if_doing_kw_and_opts_missing(asm: &mut Assembler, doing_kw_call: bool, opts_missing: i32) -> Option<()> { +fn exit_if_doing_kw_and_opts_missing(jit: &JITState, asm: &mut Assembler, doing_kw_call: bool, opts_missing: i32) -> Option<()> { // If we have unfilled optional arguments and keyword arguments then we // would need to adjust the arguments location to account for that. // For now we aren't handling this case. - exit_if(asm, doing_kw_call && opts_missing > 0, Counter::send_iseq_missing_optional_kw) + exit_if(jit, asm, doing_kw_call && opts_missing > 0, Counter::send_iseq_missing_optional_kw) } #[must_use] -fn exit_if_has_rest_and_optional_and_block(asm: &mut Assembler, iseq_has_rest: bool, opt_num: i32, iseq: *const rb_iseq_t, block_arg: bool) -> Option<()> { +fn exit_if_has_rest_and_optional_and_block(jit: &JITState, asm: &mut Assembler, iseq_has_rest: bool, opt_num: i32, iseq: *const rb_iseq_t, block_arg: bool) -> Option<()> { exit_if( + jit, asm, iseq_has_rest && opt_num != 0 && (unsafe { get_iseq_flags_has_block(iseq) } || block_arg), Counter::send_iseq_has_rest_opt_and_block @@ -8298,7 +8849,7 @@ fn exit_if_unsupported_block_arg_type( Some(Some(BlockArg::TProc)) } _ => { - gen_counter_incr(asm, Counter::send_iseq_block_arg_type); + gen_counter_incr(jit, asm, Counter::send_iseq_block_arg_type); None } } @@ -8319,7 +8870,6 @@ fn exit_if_stack_too_large(iseq: *const rb_iseq_t) -> Option<()> { fn gen_struct_aref( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, comptime_recv: VALUE, @@ -8351,7 +8901,7 @@ fn gen_struct_aref( if c_method_tracing_currently_enabled(jit) { // Struct accesses need fire c_call and c_return events, which we can't support // See :attr-tracing: - gen_counter_incr(asm, Counter::send_cfunc_tracing); + gen_counter_incr(jit, asm, Counter::send_cfunc_tracing); return None; } @@ -8381,14 +8931,12 @@ fn gen_struct_aref( let ret = asm.stack_push(Type::Unknown); asm.mov(ret, val); - jump_to_next_insn(jit, asm, ocb); - Some(EndBlock) + jump_to_next_insn(jit, asm) } fn gen_struct_aset( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, comptime_recv: VALUE, @@ -8399,10 +8947,16 @@ fn gen_struct_aset( return None; } + // If the comptime receiver is frozen, writing a struct member will raise an exception + // and we don't want to JIT code to deal with that situation. + if comptime_recv.is_frozen() { + return None; + } + if c_method_tracing_currently_enabled(jit) { // Struct accesses need fire c_call and c_return events, which we can't support // See :attr-tracing: - gen_counter_incr(asm, Counter::send_cfunc_tracing); + gen_counter_incr(jit, asm, Counter::send_cfunc_tracing); return None; } @@ -8419,6 +8973,17 @@ fn gen_struct_aset( assert!(unsafe { RB_TYPE_P(comptime_recv, RUBY_T_STRUCT) }); assert!((off as i64) < unsafe { RSTRUCT_LEN(comptime_recv) }); + // Even if the comptime recv was not frozen, future recv may be. So we need to emit a guard + // that the recv is not frozen. + // We know all structs are heap objects, so we can check the flag directly. + let recv = asm.stack_opnd(1); + let recv = asm.load(recv); + let flags = asm.load(Opnd::mem(VALUE_BITS, recv, RUBY_OFFSET_RBASIC_FLAGS)); + asm.test(flags, (RUBY_FL_FREEZE as u64).into()); + asm.jnz(Target::side_exit(Counter::opt_aset_frozen)); + + // Not frozen, so we can proceed. + asm_comment!(asm, "struct aset"); let val = asm.stack_pop(1); @@ -8429,15 +8994,13 @@ fn gen_struct_aset( let ret = asm.stack_push(Type::Unknown); asm.mov(ret, val); - jump_to_next_insn(jit, asm, ocb); - Some(EndBlock) + jump_to_next_insn(jit, asm) } // Generate code that calls a method with dynamic dispatch fn gen_send_dynamic<F: Fn(&mut Assembler) -> Opnd>( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, cd: *const rb_call_data, sp_pops: usize, vm_sendish: F, @@ -8455,12 +9018,6 @@ fn gen_send_dynamic<F: Fn(&mut Assembler) -> Opnd>( // Save PC and SP to prepare for dynamic dispatch jit_prepare_non_leaf_call(jit, asm); - // Squash stack canary that might be left over from elsewhere - assert_eq!(false, asm.get_leaf_ccall()); - if cfg!(debug_assertions) { - asm.store(asm.ctx.sp_opnd(0), 0.into()); - } - // Dispatch a method let ret = vm_sendish(asm); @@ -8474,19 +9031,17 @@ fn gen_send_dynamic<F: Fn(&mut Assembler) -> Opnd>( // Fix the interpreter SP deviated by vm_sendish asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), SP); - gen_counter_incr(asm, Counter::num_send_dynamic); + gen_counter_incr(jit, asm, Counter::num_send_dynamic); jit_perf_symbol_pop!(jit, asm, PerfMap::Codegen); // End the current block for invalidationg and sharing the same successor - jump_to_next_insn(jit, asm, ocb); - Some(EndBlock) + jump_to_next_insn(jit, asm) } fn gen_send_general( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, cd: *const rb_call_data, block: Option<BlockHandler>, ) -> Option<CodegenStatus> { @@ -8506,9 +9061,16 @@ fn gen_send_general( let mut flags = unsafe { vm_ci_flag(ci) }; // Defer compilation so we can specialize on class of receiver - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); + } + + let ci_flags = unsafe { vm_ci_flag(ci) }; + + // Dynamic stack layout. No good way to support without inlining. + if ci_flags & VM_CALL_FORWARDING != 0 { + gen_counter_incr(jit, asm, Counter::send_forwarding); + return None; } let recv_idx = argc + if flags & VM_CALL_ARGS_BLOCKARG != 0 { 1 } else { 0 }; @@ -8524,7 +9086,7 @@ fn gen_send_general( && comptime_recv != unsafe { rb_vm_top_self() } && !unsafe { RB_TYPE_P(comptime_recv, RUBY_T_CLASS) } && !unsafe { RB_TYPE_P(comptime_recv, RUBY_T_MODULE) } { - gen_counter_incr(asm, Counter::send_singleton_class); + gen_counter_incr(jit, asm, Counter::send_singleton_class); return None; } @@ -8533,28 +9095,25 @@ fn gen_send_general( let recv_opnd: YARVOpnd = recv.into(); // Log the name of the method we're calling to - #[cfg(feature = "disasm")] asm_comment!(asm, "call to {}", get_method_name(Some(comptime_recv_klass), mid)); // Gather some statistics about sends - gen_counter_incr(asm, Counter::num_send); + gen_counter_incr(jit, asm, Counter::num_send); if let Some(_known_klass) = asm.ctx.get_opnd_type(recv_opnd).known_class() { - gen_counter_incr(asm, Counter::num_send_known_class); + gen_counter_incr(jit, asm, Counter::num_send_known_class); } if asm.ctx.get_chain_depth() > 1 { - gen_counter_incr(asm, Counter::num_send_polymorphic); + gen_counter_incr(jit, asm, Counter::num_send_polymorphic); } // If megamorphic, let the caller fallback to dynamic dispatch if asm.ctx.get_chain_depth() >= SEND_MAX_DEPTH { - gen_counter_incr(asm, Counter::send_megamorphic); + gen_counter_incr(jit, asm, Counter::send_megamorphic); return None; } perf_call!("gen_send_general: ", jit_guard_known_klass( jit, asm, - ocb, - comptime_recv_klass, recv, recv_opnd, comptime_recv, @@ -8565,7 +9124,7 @@ fn gen_send_general( // Do method lookup let mut cme = unsafe { rb_callable_method_entry(comptime_recv_klass, mid) }; if cme.is_null() { - gen_counter_incr(asm, Counter::send_cme_not_found); + gen_counter_incr(jit, asm, Counter::send_cme_not_found); return None; } @@ -8582,7 +9141,7 @@ fn gen_send_general( if flags & VM_CALL_FCALL == 0 { // Can only call private methods with FCALL callsites. // (at the moment they are callsites without a receiver or an explicit `self` receiver) - gen_counter_incr(asm, Counter::send_private_not_fcall); + gen_counter_incr(jit, asm, Counter::send_private_not_fcall); return None; } } @@ -8601,7 +9160,7 @@ fn gen_send_general( // Register block for invalidation //assert!(cme->called_id == mid); - jit.assume_method_lookup_stable(asm, ocb, cme); + jit.assume_method_lookup_stable(asm, cme); // To handle the aliased method case (VM_METHOD_TYPE_ALIAS) loop { @@ -8611,13 +9170,12 @@ fn gen_send_general( VM_METHOD_TYPE_ISEQ => { let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; let frame_type = VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL; - return perf_call! { gen_send_iseq(jit, asm, ocb, iseq, ci, frame_type, None, cme, block, flags, argc, None) }; + return perf_call! { gen_send_iseq(jit, asm, iseq, ci, frame_type, None, cme, block, flags, argc, None) }; } VM_METHOD_TYPE_CFUNC => { return perf_call! { gen_send_cfunc( jit, asm, - ocb, ci, cme, block, @@ -8629,7 +9187,7 @@ fn gen_send_general( VM_METHOD_TYPE_IVAR => { // This is a .send call not supported right now for attr_reader if flags & VM_CALL_OPT_SEND != 0 { - gen_counter_incr(asm, Counter::send_send_attr_reader); + gen_counter_incr(jit, asm, Counter::send_send_attr_reader); return None; } @@ -8641,7 +9199,7 @@ fn gen_send_general( asm.stack_pop(1); } _ => { - gen_counter_incr(asm, Counter::send_getter_block_arg); + gen_counter_incr(jit, asm, Counter::send_getter_block_arg); return None; } } @@ -8661,7 +9219,7 @@ fn gen_send_general( asm.stack_pop(1); } else { // Argument count mismatch. Getters take no arguments. - gen_counter_incr(asm, Counter::send_getter_arity); + gen_counter_incr(jit, asm, Counter::send_getter_arity); return None; } } @@ -8674,7 +9232,7 @@ fn gen_send_general( // "method" we are calling into never enables those tracing // events. We are never inside the code that needs to be // invalidated when invalidation happens. - gen_counter_incr(asm, Counter::send_cfunc_tracing); + gen_counter_incr(jit, asm, Counter::send_cfunc_tracing); return None; } @@ -8684,7 +9242,6 @@ fn gen_send_general( return gen_get_ivar( jit, asm, - ocb, SEND_MAX_DEPTH, comptime_recv, ivar_name, @@ -8695,39 +9252,39 @@ fn gen_send_general( VM_METHOD_TYPE_ATTRSET => { // This is a .send call not supported right now for attr_writer if flags & VM_CALL_OPT_SEND != 0 { - gen_counter_incr(asm, Counter::send_send_attr_writer); + gen_counter_incr(jit, asm, Counter::send_send_attr_writer); return None; } if flags & VM_CALL_ARGS_SPLAT != 0 { - gen_counter_incr(asm, Counter::send_args_splat_attrset); + gen_counter_incr(jit, asm, Counter::send_args_splat_attrset); return None; } if flags & VM_CALL_KWARG != 0 { - gen_counter_incr(asm, Counter::send_attrset_kwargs); + gen_counter_incr(jit, asm, Counter::send_attrset_kwargs); return None; } else if argc != 1 || unsafe { !RB_TYPE_P(comptime_recv, RUBY_T_OBJECT) } { - gen_counter_incr(asm, Counter::send_ivar_set_method); + gen_counter_incr(jit, asm, Counter::send_ivar_set_method); return None; } else if c_method_tracing_currently_enabled(jit) { // Can't generate code for firing c_call and c_return events // See :attr-tracing: - gen_counter_incr(asm, Counter::send_cfunc_tracing); + gen_counter_incr(jit, asm, Counter::send_cfunc_tracing); return None; } else if flags & VM_CALL_ARGS_BLOCKARG != 0 { - gen_counter_incr(asm, Counter::send_attrset_block_arg); + gen_counter_incr(jit, asm, Counter::send_attrset_block_arg); return None; } else { let ivar_name = unsafe { get_cme_def_body_attr_id(cme) }; - return gen_set_ivar(jit, asm, ocb, comptime_recv, ivar_name, StackOpnd(1), None); + return gen_set_ivar(jit, asm, comptime_recv, ivar_name, StackOpnd(1), None); } } // Block method, e.g. define_method(:foo) { :my_block } VM_METHOD_TYPE_BMETHOD => { if flags & VM_CALL_ARGS_SPLAT != 0 { - gen_counter_incr(asm, Counter::send_args_splat_bmethod); + gen_counter_incr(jit, asm, Counter::send_args_splat_bmethod); return None; } - return gen_send_bmethod(jit, asm, ocb, ci, cme, block, flags, argc); + return gen_send_bmethod(jit, asm, ci, cme, block, flags, argc); } VM_METHOD_TYPE_ALIAS => { // Retrieve the aliased method and re-enter the switch @@ -8737,7 +9294,7 @@ fn gen_send_general( // Send family of methods, e.g. call/apply VM_METHOD_TYPE_OPTIMIZED => { if flags & VM_CALL_ARGS_BLOCKARG != 0 { - gen_counter_incr(asm, Counter::send_optimized_block_arg); + gen_counter_incr(jit, asm, Counter::send_optimized_block_arg); return None; } @@ -8755,12 +9312,12 @@ fn gen_send_general( // currently work, we can't do stack manipulation until we will no longer // side exit. if flags & VM_CALL_OPT_SEND != 0 { - gen_counter_incr(asm, Counter::send_send_nested); + gen_counter_incr(jit, asm, Counter::send_send_nested); return None; } if argc == 0 { - gen_counter_incr(asm, Counter::send_send_wrong_args); + gen_counter_incr(jit, asm, Counter::send_send_wrong_args); return None; } @@ -8771,19 +9328,19 @@ fn gen_send_general( mid = unsafe { rb_get_symbol_id(compile_time_name) }; if mid == 0 { // This also rejects method names that need conversion - gen_counter_incr(asm, Counter::send_send_null_mid); + gen_counter_incr(jit, asm, Counter::send_send_null_mid); return None; } cme = unsafe { rb_callable_method_entry(comptime_recv_klass, mid) }; if cme.is_null() { - gen_counter_incr(asm, Counter::send_send_null_cme); + gen_counter_incr(jit, asm, Counter::send_send_null_cme); return None; } flags |= VM_CALL_FCALL | VM_CALL_OPT_SEND; - jit.assume_method_lookup_stable(asm, ocb, cme); + jit.assume_method_lookup_stable(asm, cme); asm_comment!( asm, @@ -8799,7 +9356,6 @@ fn gen_send_general( JCC_JNE, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::guard_send_send_name_chain, ); @@ -8810,26 +9366,18 @@ fn gen_send_general( } OPTIMIZED_METHOD_TYPE_CALL => { - if block.is_some() { - gen_counter_incr(asm, Counter::send_call_block); + gen_counter_incr(jit, asm, Counter::send_call_block); return None; } if flags & VM_CALL_KWARG != 0 { - gen_counter_incr(asm, Counter::send_call_kwarg); + gen_counter_incr(jit, asm, Counter::send_call_kwarg); return None; } if flags & VM_CALL_ARGS_SPLAT != 0 { - gen_counter_incr(asm, Counter::send_args_splat_opt_call); - return None; - } - - // Optimize for single ractor mode and avoid runtime check for - // "defined with an un-shareable Proc in a different Ractor" - if !assume_single_ractor_mode(jit, asm, ocb) { - gen_counter_incr(asm, Counter::send_call_multi_ractor); + gen_counter_incr(jit, asm, Counter::send_args_splat_opt_call); return None; } @@ -8862,22 +9410,22 @@ fn gen_send_general( let stack_ret = asm.stack_push(Type::Unknown); asm.mov(stack_ret, ret); - return Some(KeepCompiling); + // End the block to allow invalidating the next instruction + return jump_to_next_insn(jit, asm); } OPTIMIZED_METHOD_TYPE_BLOCK_CALL => { - gen_counter_incr(asm, Counter::send_optimized_method_block_call); + gen_counter_incr(jit, asm, Counter::send_optimized_method_block_call); return None; } OPTIMIZED_METHOD_TYPE_STRUCT_AREF => { if flags & VM_CALL_ARGS_SPLAT != 0 { - gen_counter_incr(asm, Counter::send_args_splat_aref); + gen_counter_incr(jit, asm, Counter::send_args_splat_aref); return None; } return gen_struct_aref( jit, asm, - ocb, ci, cme, comptime_recv, @@ -8887,13 +9435,12 @@ fn gen_send_general( } OPTIMIZED_METHOD_TYPE_STRUCT_ASET => { if flags & VM_CALL_ARGS_SPLAT != 0 { - gen_counter_incr(asm, Counter::send_args_splat_aset); + gen_counter_incr(jit, asm, Counter::send_args_splat_aset); return None; } return gen_struct_aset( jit, asm, - ocb, ci, cme, comptime_recv, @@ -8907,23 +9454,23 @@ fn gen_send_general( } } VM_METHOD_TYPE_ZSUPER => { - gen_counter_incr(asm, Counter::send_zsuper_method); + gen_counter_incr(jit, asm, Counter::send_zsuper_method); return None; } VM_METHOD_TYPE_UNDEF => { - gen_counter_incr(asm, Counter::send_undef_method); + gen_counter_incr(jit, asm, Counter::send_undef_method); return None; } VM_METHOD_TYPE_NOTIMPLEMENTED => { - gen_counter_incr(asm, Counter::send_not_implemented_method); + gen_counter_incr(jit, asm, Counter::send_not_implemented_method); return None; } VM_METHOD_TYPE_MISSING => { - gen_counter_incr(asm, Counter::send_missing_method); + gen_counter_incr(jit, asm, Counter::send_missing_method); return None; } VM_METHOD_TYPE_REFINED => { - gen_counter_incr(asm, Counter::send_refined_method); + gen_counter_incr(jit, asm, Counter::send_refined_method); return None; } _ => { @@ -8935,7 +9482,10 @@ fn gen_send_general( /// Get class name from a class pointer. fn get_class_name(class: Option<VALUE>) -> String { - class.and_then(|class| unsafe { + class.filter(|&class| { + // type checks for rb_class2name() + unsafe { RB_TYPE_P(class, RUBY_T_MODULE) || RB_TYPE_P(class, RUBY_T_CLASS) } + }).and_then(|class| unsafe { cstr_to_rust_string(rb_class2name(class)) }).unwrap_or_else(|| "Unknown".to_string()) } @@ -8988,16 +9538,15 @@ fn handle_opt_send_shift_stack(asm: &mut Assembler, argc: i32) { fn gen_opt_send_without_block( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Generate specialized code if possible let cd = jit.get_arg(0).as_ptr(); - if let Some(status) = perf_call! { gen_send_general(jit, asm, ocb, cd, None) } { + if let Some(status) = perf_call! { gen_send_general(jit, asm, cd, None) } { return Some(status); } // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of send - gen_send_dynamic(jit, asm, ocb, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { + gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { extern "C" { fn rb_vm_opt_send_without_block(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; } @@ -9011,18 +9560,17 @@ fn gen_opt_send_without_block( fn gen_send( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Generate specialized code if possible let cd = jit.get_arg(0).as_ptr(); let block = jit.get_arg(1).as_optional_ptr().map(|iseq| BlockHandler::BlockISeq(iseq)); - if let Some(status) = perf_call! { gen_send_general(jit, asm, ocb, cd, block) } { + if let Some(status) = perf_call! { gen_send_general(jit, asm, cd, block) } { return Some(status); } // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of send let blockiseq = jit.get_arg(1).as_iseq(); - gen_send_dynamic(jit, asm, ocb, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { + gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { extern "C" { fn rb_vm_send(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; } @@ -9033,19 +9581,42 @@ fn gen_send( }) } +fn gen_sendforward( + jit: &mut JITState, + asm: &mut Assembler, +) -> Option<CodegenStatus> { + // Generate specialized code if possible + let cd = jit.get_arg(0).as_ptr(); + let block = jit.get_arg(1).as_optional_ptr().map(|iseq| BlockHandler::BlockISeq(iseq)); + if let Some(status) = perf_call! { gen_send_general(jit, asm, cd, block) } { + return Some(status); + } + + // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of sendforward + let blockiseq = jit.get_arg(1).as_iseq(); + gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { + extern "C" { + fn rb_vm_sendforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm.ccall( + rb_vm_sendforward as *const u8, + vec![EC, CFP, (cd as usize).into(), VALUE(blockiseq as usize).into()], + ) + }) +} + fn gen_invokeblock( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Generate specialized code if possible let cd = jit.get_arg(0).as_ptr(); - if let Some(status) = gen_invokeblock_specialized(jit, asm, ocb, cd) { + if let Some(status) = gen_invokeblock_specialized(jit, asm, cd) { return Some(status); } // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of send - gen_send_dynamic(jit, asm, ocb, cd, unsafe { rb_yjit_invokeblock_sp_pops((*cd).ci) }, |asm| { + gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_invokeblock_sp_pops((*cd).ci) }, |asm| { extern "C" { fn rb_vm_invokeblock(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; } @@ -9059,17 +9630,15 @@ fn gen_invokeblock( fn gen_invokeblock_specialized( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, cd: *const rb_call_data, ) -> Option<CodegenStatus> { - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } // Fallback to dynamic dispatch if this callsite is megamorphic if asm.ctx.get_chain_depth() >= SEND_MAX_DEPTH { - gen_counter_incr(asm, Counter::invokeblock_megamorphic); + gen_counter_incr(jit, asm, Counter::invokeblock_megamorphic); return None; } @@ -9085,7 +9654,7 @@ fn gen_invokeblock_specialized( // Handle each block_handler type if comptime_handler.0 == VM_BLOCK_HANDLER_NONE as usize { // no block given - gen_counter_incr(asm, Counter::invokeblock_none); + gen_counter_incr(jit, asm, Counter::invokeblock_none); None } else if comptime_handler.0 & 0x3 == 0x1 { // VM_BH_ISEQ_BLOCK_P asm_comment!(asm, "get local EP"); @@ -9101,15 +9670,14 @@ fn gen_invokeblock_specialized( JCC_JNE, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::guard_invokeblock_tag_changed, ); // If the current ISEQ is annotated to be inlined but it's not being inlined here, // generate a dynamic dispatch to avoid making this yield megamorphic. - if unsafe { rb_yjit_iseq_builtin_attrs(jit.iseq) } & BUILTIN_ATTR_INLINE_BLOCK != 0 && !asm.ctx.inline() { - gen_counter_incr(asm, Counter::invokeblock_iseq_not_inlined); + if unsafe { rb_jit_iseq_builtin_attrs(jit.iseq) } & BUILTIN_ATTR_INLINE_BLOCK != 0 && !asm.ctx.inline() { + gen_counter_incr(jit, asm, Counter::invokeblock_iseq_not_inlined); return None; } @@ -9124,20 +9692,19 @@ fn gen_invokeblock_specialized( JCC_JNE, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::guard_invokeblock_iseq_block_changed, ); - perf_call! { gen_send_iseq(jit, asm, ocb, comptime_iseq, ci, VM_FRAME_MAGIC_BLOCK, None, 0 as _, None, flags, argc, Some(captured_opnd)) } + perf_call! { gen_send_iseq(jit, asm, comptime_iseq, ci, VM_FRAME_MAGIC_BLOCK, None, 0 as _, None, flags, argc, Some(captured_opnd)) } } else if comptime_handler.0 & 0x3 == 0x3 { // VM_BH_IFUNC_P // We aren't handling CALLER_SETUP_ARG and CALLER_REMOVE_EMPTY_KW_SPLAT yet. if flags & VM_CALL_ARGS_SPLAT != 0 { - gen_counter_incr(asm, Counter::invokeblock_ifunc_args_splat); + gen_counter_incr(jit, asm, Counter::invokeblock_ifunc_args_splat); return None; } if flags & VM_CALL_KW_SPLAT != 0 { - gen_counter_incr(asm, Counter::invokeblock_ifunc_kw_splat); + gen_counter_incr(jit, asm, Counter::invokeblock_ifunc_kw_splat); return None; } @@ -9154,7 +9721,6 @@ fn gen_invokeblock_specialized( JCC_JNE, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::guard_invokeblock_tag_changed, ); @@ -9181,13 +9747,12 @@ fn gen_invokeblock_specialized( asm.clear_local_types(); // Share the successor with other chains - jump_to_next_insn(jit, asm, ocb); - Some(EndBlock) + jump_to_next_insn(jit, asm) } else if comptime_handler.symbol_p() { - gen_counter_incr(asm, Counter::invokeblock_symbol); + gen_counter_incr(jit, asm, Counter::invokeblock_symbol); None } else { // Proc - gen_counter_incr(asm, Counter::invokeblock_proc); + gen_counter_incr(jit, asm, Counter::invokeblock_proc); None } } @@ -9195,17 +9760,16 @@ fn gen_invokeblock_specialized( fn gen_invokesuper( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Generate specialized code if possible let cd = jit.get_arg(0).as_ptr(); - if let Some(status) = gen_invokesuper_specialized(jit, asm, ocb, cd) { + if let Some(status) = gen_invokesuper_specialized(jit, asm, cd) { return Some(status); } - // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of send + // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of invokesuper let blockiseq = jit.get_arg(1).as_iseq(); - gen_send_dynamic(jit, asm, ocb, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { + gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { extern "C" { fn rb_vm_invokesuper(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; } @@ -9216,16 +9780,37 @@ fn gen_invokesuper( }) } +fn gen_invokesuperforward( + jit: &mut JITState, + asm: &mut Assembler, +) -> Option<CodegenStatus> { + // Generate specialized code if possible + let cd = jit.get_arg(0).as_ptr(); + if let Some(status) = gen_invokesuper_specialized(jit, asm, cd) { + return Some(status); + } + + // Otherwise, fallback to dynamic dispatch using the interpreter's implementation of invokesuperforward + let blockiseq = jit.get_arg(1).as_iseq(); + gen_send_dynamic(jit, asm, cd, unsafe { rb_yjit_sendish_sp_pops((*cd).ci) }, |asm| { + extern "C" { + fn rb_vm_invokesuperforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm.ccall( + rb_vm_invokesuperforward as *const u8, + vec![EC, CFP, (cd as usize).into(), VALUE(blockiseq as usize).into()], + ) + }) +} + fn gen_invokesuper_specialized( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, cd: *const rb_call_data, ) -> Option<CodegenStatus> { // Defer compilation so we can specialize on class of receiver - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } // Handle the last two branches of vm_caller_setup_arg_block @@ -9237,13 +9822,13 @@ fn gen_invokesuper_specialized( // Fallback to dynamic dispatch if this callsite is megamorphic if asm.ctx.get_chain_depth() >= SEND_MAX_DEPTH { - gen_counter_incr(asm, Counter::invokesuper_megamorphic); + gen_counter_incr(jit, asm, Counter::invokesuper_megamorphic); return None; } let me = unsafe { rb_vm_frame_method_entry(jit.get_cfp()) }; if me.is_null() { - gen_counter_incr(asm, Counter::invokesuper_no_me); + gen_counter_incr(jit, asm, Counter::invokesuper_no_me); return None; } @@ -9256,7 +9841,7 @@ fn gen_invokesuper_specialized( if current_defined_class.builtin_type() == RUBY_T_ICLASS && unsafe { RB_TYPE_P((*rbasic_ptr).klass, RUBY_T_MODULE) && FL_TEST_RAW((*rbasic_ptr).klass, VALUE(RMODULE_IS_REFINEMENT.as_usize())) != VALUE(0) } { - gen_counter_incr(asm, Counter::invokesuper_refinement); + gen_counter_incr(jit, asm, Counter::invokesuper_refinement); return None; } let comptime_superclass = @@ -9271,11 +9856,15 @@ fn gen_invokesuper_specialized( // Note, not using VM_CALL_ARGS_SIMPLE because sometimes we pass a block. if ci_flags & VM_CALL_KWARG != 0 { - gen_counter_incr(asm, Counter::invokesuper_kwarg); + gen_counter_incr(jit, asm, Counter::invokesuper_kwarg); return None; } if ci_flags & VM_CALL_KW_SPLAT != 0 { - gen_counter_incr(asm, Counter::invokesuper_kw_splat); + gen_counter_incr(jit, asm, Counter::invokesuper_kw_splat); + return None; + } + if ci_flags & VM_CALL_FORWARDING != 0 { + gen_counter_incr(jit, asm, Counter::invokesuper_forwarding); return None; } @@ -9286,20 +9875,20 @@ fn gen_invokesuper_specialized( // check and side exit. let comptime_recv = jit.peek_at_stack(&asm.ctx, argc as isize); if unsafe { rb_obj_is_kind_of(comptime_recv, current_defined_class) } == VALUE(0) { - gen_counter_incr(asm, Counter::invokesuper_defined_class_mismatch); + gen_counter_incr(jit, asm, Counter::invokesuper_defined_class_mismatch); return None; } // Don't compile `super` on objects with singleton class to avoid retaining the receiver. if VALUE(0) != unsafe { FL_TEST(comptime_recv.class_of(), VALUE(RUBY_FL_SINGLETON as usize)) } { - gen_counter_incr(asm, Counter::invokesuper_singleton_class); + gen_counter_incr(jit, asm, Counter::invokesuper_singleton_class); return None; } // Do method lookup let cme = unsafe { rb_callable_method_entry(comptime_superclass, mid) }; if cme.is_null() { - gen_counter_incr(asm, Counter::invokesuper_no_cme); + gen_counter_incr(jit, asm, Counter::invokesuper_no_cme); return None; } @@ -9307,7 +9896,7 @@ fn gen_invokesuper_specialized( let cme_def_type = unsafe { get_cme_def_type(cme) }; if cme_def_type != VM_METHOD_TYPE_ISEQ && cme_def_type != VM_METHOD_TYPE_CFUNC { // others unimplemented - gen_counter_incr(asm, Counter::invokesuper_not_iseq_or_cfunc); + gen_counter_incr(jit, asm, Counter::invokesuper_not_iseq_or_cfunc); return None; } @@ -9325,15 +9914,14 @@ fn gen_invokesuper_specialized( JCC_JNE, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::guard_invokesuper_me_changed, ); // We need to assume that both our current method entry and the super // method entry we invoke remain stable - jit.assume_method_lookup_stable(asm, ocb, me); - jit.assume_method_lookup_stable(asm, ocb, cme); + jit.assume_method_lookup_stable(asm, me); + jit.assume_method_lookup_stable(asm, cme); // Method calls may corrupt types asm.clear_local_types(); @@ -9342,10 +9930,10 @@ fn gen_invokesuper_specialized( VM_METHOD_TYPE_ISEQ => { let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; let frame_type = VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL; - perf_call! { gen_send_iseq(jit, asm, ocb, iseq, ci, frame_type, None, cme, Some(block), ci_flags, argc, None) } + perf_call! { gen_send_iseq(jit, asm, iseq, ci, frame_type, None, cme, Some(block), ci_flags, argc, None) } } VM_METHOD_TYPE_CFUNC => { - perf_call! { gen_send_cfunc(jit, asm, ocb, ci, cme, Some(block), None, ci_flags, argc) } + perf_call! { gen_send_cfunc(jit, asm, ci, cme, Some(block), None, ci_flags, argc) } } _ => unreachable!(), } @@ -9354,7 +9942,6 @@ fn gen_invokesuper_specialized( fn gen_leave( _jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Only the return value should be on the stack assert_eq!(1, asm.ctx.get_stack_size(), "leave instruction expects stack size 1, but was: {}", asm.ctx.get_stack_size()); @@ -9367,7 +9954,7 @@ fn gen_leave( asm_comment!(asm, "pop stack frame"); let incr_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); asm.mov(CFP, incr_cfp); - asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP); + asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); // Load the return value let retval_opnd = asm.stack_pop(1); @@ -9391,7 +9978,6 @@ fn gen_leave( fn gen_getglobal( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let gid = jit.get_arg(0).as_usize(); @@ -9412,7 +9998,6 @@ fn gen_getglobal( fn gen_setglobal( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let gid = jit.get_arg(0).as_usize(); @@ -9436,7 +10021,6 @@ fn gen_setglobal( fn gen_anytostring( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Save the PC and SP since we might call #to_s jit_prepare_non_leaf_call(jit, asm); @@ -9457,11 +10041,9 @@ fn gen_anytostring( fn gen_objtostring( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } let recv = asm.stack_opnd(0); @@ -9471,8 +10053,6 @@ fn gen_objtostring( jit_guard_known_klass( jit, asm, - ocb, - comptime_recv.class_of(), recv, recv.into(), comptime_recv, @@ -9482,16 +10062,43 @@ fn gen_objtostring( // No work needed. The string value is already on the top of the stack. Some(KeepCompiling) + } else if unsafe { RB_TYPE_P(comptime_recv, RUBY_T_SYMBOL) } && assume_method_basic_definition(jit, asm, comptime_recv.class_of(), ID!(to_s)) { + jit_guard_known_klass( + jit, + asm, + recv, + recv.into(), + comptime_recv, + SEND_MAX_DEPTH, + Counter::objtostring_not_string, + ); + + extern "C" { + fn rb_sym2str(sym: VALUE) -> VALUE; + } + + // Same optimization done in the interpreter: rb_sym_to_s() allocates a mutable string, but since we are only + // going to use this string for interpolation, it's fine to use the + // frozen string. + // rb_sym2str does not allocate. + let sym = recv; + let str = asm.ccall(rb_sym2str as *const u8, vec![sym]); + asm.stack_pop(1); + + // Push the return value + let stack_ret = asm.stack_push(Type::TString); + asm.mov(stack_ret, str); + + Some(KeepCompiling) } else { let cd = jit.get_arg(0).as_ptr(); - perf_call! { gen_send_general(jit, asm, ocb, cd, None) } + perf_call! { gen_send_general(jit, asm, cd, None) } } } fn gen_intern( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // Save the PC and SP because we might allocate jit_prepare_call_with_gc(jit, asm); @@ -9510,7 +10117,6 @@ fn gen_intern( fn gen_toregexp( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let opt = jit.get_arg(0).as_i64(); let cnt = jit.get_arg(1).as_usize(); @@ -9561,7 +10167,6 @@ fn gen_toregexp( fn gen_getspecial( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // This takes two arguments, key and type // key is only used when type == 0 @@ -9637,7 +10242,6 @@ fn gen_getspecial( fn gen_getclassvariable( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // rb_vm_getclassvariable can raise exceptions. jit_prepare_non_leaf_call(jit, asm); @@ -9645,7 +10249,7 @@ fn gen_getclassvariable( let val_opnd = asm.ccall( rb_vm_getclassvariable as *const u8, vec![ - Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), + VALUE(jit.iseq as usize).into(), CFP, Opnd::UImm(jit.get_arg(0).as_u64()), Opnd::UImm(jit.get_arg(1).as_u64()), @@ -9661,7 +10265,6 @@ fn gen_getclassvariable( fn gen_setclassvariable( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // rb_vm_setclassvariable can raise exceptions. jit_prepare_non_leaf_call(jit, asm); @@ -9670,7 +10273,7 @@ fn gen_setclassvariable( asm.ccall( rb_vm_setclassvariable as *const u8, vec![ - Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), + VALUE(jit.iseq as usize).into(), CFP, Opnd::UImm(jit.get_arg(0).as_u64()), val, @@ -9685,7 +10288,6 @@ fn gen_setclassvariable( fn gen_getconstant( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let id = jit.get_arg(0).as_usize(); @@ -9720,7 +10322,6 @@ fn gen_getconstant( fn gen_opt_getconstant_path( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let const_cache_as_value = jit.get_arg(0); let ic: *const iseq_inline_constant_cache = const_cache_as_value.as_ptr(); @@ -9728,7 +10329,7 @@ fn gen_opt_getconstant_path( // Make sure there is an exit for this block as the interpreter might want // to invalidate this block from yjit_constant_ic_update(). - jit_ensure_block_entry_exit(jit, asm, ocb)?; + jit_ensure_block_entry_exit(jit, asm)?; // See vm_ic_hit_p(). The same conditions are checked in yjit_constant_ic_update(). // If a cache is not filled, fallback to the general C call. @@ -9749,15 +10350,19 @@ fn gen_opt_getconstant_path( let stack_top = asm.stack_push(Type::Unknown); asm.store(stack_top, val); - jump_to_next_insn(jit, asm, ocb); - return Some(EndBlock); + return jump_to_next_insn(jit, asm); } - if !unsafe { (*ice).ic_cref }.is_null() { + let cref_sensitive = !unsafe { (*ice).ic_cref }.is_null(); + let is_shareable = unsafe { rb_yjit_constcache_shareable(ice) }; + let needs_checks = cref_sensitive || (!is_shareable && !assume_single_ractor_mode(jit, asm)); + + if needs_checks { // Cache is keyed on a certain lexical scope. Use the interpreter's cache. let inline_cache = asm.load(Opnd::const_ptr(ic as *const u8)); // Call function to verify the cache. It doesn't allocate or call methods. + // This includes a check for Ractor safety let ret_val = asm.ccall( rb_vm_ic_hit_p as *const u8, vec![inline_cache, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP)] @@ -9786,21 +10391,14 @@ fn gen_opt_getconstant_path( let stack_top = asm.stack_push(Type::Unknown); asm.store(stack_top, ic_entry_val); } else { - // Optimize for single ractor mode. - if !assume_single_ractor_mode(jit, asm, ocb) { - gen_counter_incr(asm, Counter::opt_getconstant_path_multi_ractor); - return None; - } - // Invalidate output code on any constant writes associated with // constants referenced within the current block. - jit.assume_stable_constant_names(asm, ocb, idlist); + jit.assume_stable_constant_names(asm, idlist); jit_putobject(asm, unsafe { (*ice).value }); } - jump_to_next_insn(jit, asm, ocb); - Some(EndBlock) + jump_to_next_insn(jit, asm) } // Push the explicit block parameter onto the temporary stack. Part of the @@ -9809,11 +10407,9 @@ fn gen_opt_getconstant_path( fn gen_getblockparamproxy( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { - if !jit.at_current_insn() { - defer_compilation(jit, asm, ocb); - return Some(EndBlock); + if !jit.at_compile_target() { + return jit.defer_compilation(asm); } // EP level @@ -9829,7 +10425,7 @@ fn gen_getblockparamproxy( unsafe { rb_obj_is_proc(comptime_handler) }.test() // block is a Proc ) { // Missing the symbol case, where we basically need to call Symbol#to_proc at runtime - gen_counter_incr(asm, Counter::gbpp_unsupported_type); + gen_counter_incr(jit, asm, Counter::gbpp_unsupported_type); return None; } @@ -9865,7 +10461,6 @@ fn gen_getblockparamproxy( JCC_JNZ, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::gbpp_block_handler_not_none, ); @@ -9885,7 +10480,6 @@ fn gen_getblockparamproxy( JCC_JZ, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::gbpp_block_handler_not_iseq, ); @@ -9919,7 +10513,6 @@ fn gen_getblockparamproxy( JCC_JE, jit, asm, - ocb, SEND_MAX_DEPTH, Counter::gbpp_block_handler_not_proc, ); @@ -9930,22 +10523,19 @@ fn gen_getblockparamproxy( unreachable!("absurd given initial filtering"); } - jump_to_next_insn(jit, asm, ocb); - - Some(EndBlock) + jump_to_next_insn(jit, asm) } fn gen_getblockparam( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { // EP level let level = jit.get_arg(1).as_u32(); // Save the PC and SP because we might allocate jit_prepare_call_with_gc(jit, asm); - asm.spill_temps(); // For ccall. Unconditionally spill them for RegTemps consistency. + asm.spill_regs(); // For ccall. Unconditionally spill them for RegMappings consistency. // A mirror of the interpreter code. Checking for the case // where it's pushing rb_block_param_proxy. @@ -10020,7 +10610,6 @@ fn gen_getblockparam( fn gen_invokebuiltin( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let bf: *const rb_builtin_function = jit.get_arg(0).as_ptr(); let bf_argc: usize = unsafe { (*bf).argc }.try_into().expect("non negative argc"); @@ -10059,7 +10648,6 @@ fn gen_invokebuiltin( fn gen_opt_invokebuiltin_delegate( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, ) -> Option<CodegenStatus> { let bf: *const rb_builtin_function = jit.get_arg(0).as_ptr(); let bf_argc = unsafe { (*bf).argc }; @@ -10110,6 +10698,7 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> { YARVINSN_dup => Some(gen_dup), YARVINSN_dupn => Some(gen_dupn), YARVINSN_swap => Some(gen_swap), + YARVINSN_opt_reverse => Some(gen_opt_reverse), YARVINSN_putnil => Some(gen_putnil), YARVINSN_putobject => Some(gen_putobject), YARVINSN_putobject_INT2FIX_0_ => Some(gen_putobject_int2fix), @@ -10140,8 +10729,11 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> { YARVINSN_opt_gt => Some(gen_opt_gt), YARVINSN_opt_ge => Some(gen_opt_ge), YARVINSN_opt_mod => Some(gen_opt_mod), + YARVINSN_opt_ary_freeze => Some(gen_opt_ary_freeze), + YARVINSN_opt_hash_freeze => Some(gen_opt_hash_freeze), YARVINSN_opt_str_freeze => Some(gen_opt_str_freeze), YARVINSN_opt_str_uminus => Some(gen_opt_str_uminus), + YARVINSN_opt_duparray_send => Some(gen_opt_duparray_send), YARVINSN_opt_newarray_send => Some(gen_opt_newarray_send), YARVINSN_splatarray => Some(gen_splatarray), YARVINSN_splatkw => Some(gen_splatkw), @@ -10164,7 +10756,6 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> { YARVINSN_opt_neq => Some(gen_opt_neq), YARVINSN_opt_aref => Some(gen_opt_aref), YARVINSN_opt_aset => Some(gen_opt_aset), - YARVINSN_opt_aref_with => Some(gen_opt_aref_with), YARVINSN_opt_mult => Some(gen_opt_mult), YARVINSN_opt_div => Some(gen_opt_div), YARVINSN_opt_ltlt => Some(gen_opt_ltlt), @@ -10186,13 +10777,16 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> { YARVINSN_branchnil => Some(gen_branchnil), YARVINSN_throw => Some(gen_throw), YARVINSN_jump => Some(gen_jump), + YARVINSN_opt_new => Some(gen_opt_new), YARVINSN_getblockparamproxy => Some(gen_getblockparamproxy), YARVINSN_getblockparam => Some(gen_getblockparam), YARVINSN_opt_send_without_block => Some(gen_opt_send_without_block), YARVINSN_send => Some(gen_send), + YARVINSN_sendforward => Some(gen_sendforward), YARVINSN_invokeblock => Some(gen_invokeblock), YARVINSN_invokesuper => Some(gen_invokesuper), + YARVINSN_invokesuperforward => Some(gen_invokesuperforward), YARVINSN_leave => Some(gen_leave), YARVINSN_getglobal => Some(gen_getglobal), @@ -10210,13 +10804,12 @@ fn get_gen_fn(opcode: VALUE) -> Option<InsnGenFn> { } } -// Return true when the codegen function generates code. -// known_recv_class has Some value when the caller has used jit_guard_known_klass(). -// See yjit_reg_method(). +/// Return true when the codegen function generates code. +/// known_recv_class has Some value when the caller has used jit_guard_known_klass(). +/// See [reg_method_codegen] type MethodGenFn = fn( jit: &mut JITState, asm: &mut Assembler, - ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, block: Option<BlockHandler>, @@ -10233,87 +10826,94 @@ pub fn yjit_reg_method_codegen_fns() { assert!(METHOD_CODEGEN_TABLE.is_none()); METHOD_CODEGEN_TABLE = Some(HashMap::default()); - // Specialization for C methods. See yjit_reg_method() for details. - yjit_reg_method(rb_cBasicObject, "!", jit_rb_obj_not); - - yjit_reg_method(rb_cNilClass, "nil?", jit_rb_true); - yjit_reg_method(rb_mKernel, "nil?", jit_rb_false); - yjit_reg_method(rb_mKernel, "is_a?", jit_rb_kernel_is_a); - yjit_reg_method(rb_mKernel, "kind_of?", jit_rb_kernel_is_a); - yjit_reg_method(rb_mKernel, "instance_of?", jit_rb_kernel_instance_of); - - yjit_reg_method(rb_cBasicObject, "==", jit_rb_obj_equal); - yjit_reg_method(rb_cBasicObject, "equal?", jit_rb_obj_equal); - yjit_reg_method(rb_cBasicObject, "!=", jit_rb_obj_not_equal); - yjit_reg_method(rb_mKernel, "eql?", jit_rb_obj_equal); - yjit_reg_method(rb_cModule, "==", jit_rb_obj_equal); - yjit_reg_method(rb_cModule, "===", jit_rb_mod_eqq); - yjit_reg_method(rb_cSymbol, "==", jit_rb_obj_equal); - yjit_reg_method(rb_cSymbol, "===", jit_rb_obj_equal); - yjit_reg_method(rb_cInteger, "==", jit_rb_int_equal); - yjit_reg_method(rb_cInteger, "===", jit_rb_int_equal); - - yjit_reg_method(rb_cInteger, "succ", jit_rb_int_succ); - yjit_reg_method(rb_cInteger, "/", jit_rb_int_div); - yjit_reg_method(rb_cInteger, "<<", jit_rb_int_lshift); - yjit_reg_method(rb_cInteger, ">>", jit_rb_int_rshift); - yjit_reg_method(rb_cInteger, "^", jit_rb_int_xor); - yjit_reg_method(rb_cInteger, "[]", jit_rb_int_aref); - - yjit_reg_method(rb_cFloat, "+", jit_rb_float_plus); - yjit_reg_method(rb_cFloat, "-", jit_rb_float_minus); - yjit_reg_method(rb_cFloat, "*", jit_rb_float_mul); - yjit_reg_method(rb_cFloat, "/", jit_rb_float_div); - - yjit_reg_method(rb_cString, "empty?", jit_rb_str_empty_p); - yjit_reg_method(rb_cString, "to_s", jit_rb_str_to_s); - yjit_reg_method(rb_cString, "to_str", jit_rb_str_to_s); - yjit_reg_method(rb_cString, "length", jit_rb_str_length); - yjit_reg_method(rb_cString, "size", jit_rb_str_length); - yjit_reg_method(rb_cString, "bytesize", jit_rb_str_bytesize); - yjit_reg_method(rb_cString, "getbyte", jit_rb_str_getbyte); - yjit_reg_method(rb_cString, "setbyte", jit_rb_str_setbyte); - yjit_reg_method(rb_cString, "byteslice", jit_rb_str_byteslice); - yjit_reg_method(rb_cString, "<<", jit_rb_str_concat); - yjit_reg_method(rb_cString, "+@", jit_rb_str_uplus); - - yjit_reg_method(rb_cNilClass, "===", jit_rb_case_equal); - yjit_reg_method(rb_cTrueClass, "===", jit_rb_case_equal); - yjit_reg_method(rb_cFalseClass, "===", jit_rb_case_equal); - - yjit_reg_method(rb_cArray, "empty?", jit_rb_ary_empty_p); - yjit_reg_method(rb_cArray, "length", jit_rb_ary_length); - yjit_reg_method(rb_cArray, "size", jit_rb_ary_length); - yjit_reg_method(rb_cArray, "<<", jit_rb_ary_push); - - yjit_reg_method(rb_cHash, "empty?", jit_rb_hash_empty_p); - - yjit_reg_method(rb_mKernel, "respond_to?", jit_obj_respond_to); - yjit_reg_method(rb_mKernel, "block_given?", jit_rb_f_block_given_p); - - yjit_reg_method(rb_cClass, "superclass", jit_rb_class_superclass); - - yjit_reg_method(rb_singleton_class(rb_cThread), "current", jit_thread_s_current); - } -} - -// Register a specialized codegen function for a particular method. Note that -// the if the function returns true, the code it generates runs without a -// control frame and without interrupt checks. To avoid creating observable -// behavior changes, the codegen function should only target simple code paths -// that do not allocate and do not make method calls. -fn yjit_reg_method(klass: VALUE, mid_str: &str, gen_fn: MethodGenFn) { - let id_string = std::ffi::CString::new(mid_str).expect("couldn't convert to CString!"); - let mid = unsafe { rb_intern(id_string.as_ptr()) }; + // Specialization for C methods. See the function's docs for details. + reg_method_codegen(rb_cBasicObject, "!", jit_rb_obj_not); + + reg_method_codegen(rb_cNilClass, "nil?", jit_rb_true); + reg_method_codegen(rb_mKernel, "nil?", jit_rb_false); + reg_method_codegen(rb_mKernel, "is_a?", jit_rb_kernel_is_a); + reg_method_codegen(rb_mKernel, "kind_of?", jit_rb_kernel_is_a); + reg_method_codegen(rb_mKernel, "instance_of?", jit_rb_kernel_instance_of); + + reg_method_codegen(rb_cBasicObject, "==", jit_rb_obj_equal); + reg_method_codegen(rb_cBasicObject, "equal?", jit_rb_obj_equal); + reg_method_codegen(rb_cBasicObject, "!=", jit_rb_obj_not_equal); + reg_method_codegen(rb_mKernel, "eql?", jit_rb_obj_equal); + reg_method_codegen(rb_cModule, "==", jit_rb_obj_equal); + reg_method_codegen(rb_cModule, "===", jit_rb_mod_eqq); + reg_method_codegen(rb_cModule, "name", jit_rb_mod_name); + reg_method_codegen(rb_cSymbol, "==", jit_rb_obj_equal); + reg_method_codegen(rb_cSymbol, "===", jit_rb_obj_equal); + reg_method_codegen(rb_cInteger, "==", jit_rb_int_equal); + reg_method_codegen(rb_cInteger, "===", jit_rb_int_equal); + + reg_method_codegen(rb_cInteger, "succ", jit_rb_int_succ); + reg_method_codegen(rb_cInteger, "pred", jit_rb_int_pred); + reg_method_codegen(rb_cInteger, "/", jit_rb_int_div); + reg_method_codegen(rb_cInteger, "<<", jit_rb_int_lshift); + reg_method_codegen(rb_cInteger, ">>", jit_rb_int_rshift); + reg_method_codegen(rb_cInteger, "^", jit_rb_int_xor); + reg_method_codegen(rb_cInteger, "[]", jit_rb_int_aref); + + reg_method_codegen(rb_cFloat, "+", jit_rb_float_plus); + reg_method_codegen(rb_cFloat, "-", jit_rb_float_minus); + reg_method_codegen(rb_cFloat, "*", jit_rb_float_mul); + reg_method_codegen(rb_cFloat, "/", jit_rb_float_div); + + reg_method_codegen(rb_cString, "dup", jit_rb_str_dup); + reg_method_codegen(rb_cString, "empty?", jit_rb_str_empty_p); + reg_method_codegen(rb_cString, "to_s", jit_rb_str_to_s); + reg_method_codegen(rb_cString, "to_str", jit_rb_str_to_s); + reg_method_codegen(rb_cString, "length", jit_rb_str_length); + reg_method_codegen(rb_cString, "size", jit_rb_str_length); + reg_method_codegen(rb_cString, "bytesize", jit_rb_str_bytesize); + reg_method_codegen(rb_cString, "getbyte", jit_rb_str_getbyte); + reg_method_codegen(rb_cString, "setbyte", jit_rb_str_setbyte); + reg_method_codegen(rb_cString, "byteslice", jit_rb_str_byteslice); + reg_method_codegen(rb_cString, "[]", jit_rb_str_aref_m); + reg_method_codegen(rb_cString, "slice", jit_rb_str_aref_m); + reg_method_codegen(rb_cString, "<<", jit_rb_str_concat); + reg_method_codegen(rb_cString, "+@", jit_rb_str_uplus); + + reg_method_codegen(rb_cNilClass, "===", jit_rb_case_equal); + reg_method_codegen(rb_cTrueClass, "===", jit_rb_case_equal); + reg_method_codegen(rb_cFalseClass, "===", jit_rb_case_equal); + + reg_method_codegen(rb_cArray, "empty?", jit_rb_ary_empty_p); + reg_method_codegen(rb_cArray, "length", jit_rb_ary_length); + reg_method_codegen(rb_cArray, "size", jit_rb_ary_length); + reg_method_codegen(rb_cArray, "<<", jit_rb_ary_push); + + reg_method_codegen(rb_cHash, "empty?", jit_rb_hash_empty_p); + + reg_method_codegen(rb_mKernel, "respond_to?", jit_obj_respond_to); + reg_method_codegen(rb_mKernel, "block_given?", jit_rb_f_block_given_p); + reg_method_codegen(rb_mKernel, "dup", jit_rb_obj_dup); + + reg_method_codegen(rb_cClass, "superclass", jit_rb_class_superclass); + + reg_method_codegen(rb_singleton_class(rb_cThread), "current", jit_thread_s_current); + } +} + +/// Register a specialized codegen function for a particular method. Note that +/// if the function returns true, the code it generates runs without a +/// control frame and without interrupt checks, completely substituting the +/// original implementation of the method. To avoid creating observable +/// behavior changes, prefer targeting simple code paths that do not allocate +/// and do not make method calls. +/// +/// See also: [lookup_cfunc_codegen]. +fn reg_method_codegen(klass: VALUE, method_name: &str, gen_fn: MethodGenFn) { + let mid = unsafe { rb_intern2(method_name.as_ptr().cast(), method_name.len().try_into().unwrap()) }; let me = unsafe { rb_method_entry_at(klass, mid) }; if me.is_null() { - panic!("undefined optimized method!: {mid_str}"); + panic!("undefined optimized method!: {method_name}"); } - // For now, only cfuncs are supported - //RUBY_ASSERT(me && me->def); - //RUBY_ASSERT(me->def->type == VM_METHOD_TYPE_CFUNC); + // For now, only cfuncs are supported (me->cme cast fine since it's just me->def->type). + debug_assert_eq!(VM_METHOD_TYPE_CFUNC, unsafe { get_cme_def_type(me.cast()) }); let method_serial = unsafe { let def = (*me).def; @@ -10323,8 +10923,15 @@ fn yjit_reg_method(klass: VALUE, mid_str: &str, gen_fn: MethodGenFn) { unsafe { METHOD_CODEGEN_TABLE.as_mut().unwrap().insert(method_serial, gen_fn); } } +pub fn yjit_shutdown_free_codegen_table() { + unsafe { METHOD_CODEGEN_TABLE = None; }; +} + /// Global state needed for code generation pub struct CodegenGlobals { + /// Flat vector of bits to store compressed context data + context_data: BitVector, + /// Inline code block (fast path) inline_cb: CodeBlock, @@ -10376,11 +10983,11 @@ impl CodegenGlobals { /// Initialize the codegen globals pub fn init() { // Executable memory and code page size in bytes - let mem_size = get_option!(exec_mem_size); + let exec_mem_size = get_option!(exec_mem_size).unwrap_or(get_option!(mem_size)); #[cfg(not(test))] let (mut cb, mut ocb) = { - let virt_block: *mut u8 = unsafe { rb_yjit_reserve_addr_space(mem_size as u32) }; + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(exec_mem_size as u32) }; // Memory protection syscalls need page-aligned addresses, so check it here. Assuming // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the @@ -10389,7 +10996,7 @@ impl CodegenGlobals { // // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. - let page_size = unsafe { rb_yjit_get_page_size() }; + let page_size = unsafe { rb_jit_get_page_size() }; assert_eq!( virt_block as usize % page_size.as_usize(), 0, "Start of virtual address block should be page-aligned", @@ -10402,13 +11009,16 @@ impl CodegenGlobals { SystemAllocator {}, page_size, NonNull::new(virt_block).unwrap(), - mem_size, + exec_mem_size, + get_option!(mem_size), ); - let mem_block = Rc::new(RefCell::new(mem_block)); + let mem_block = Rc::new(mem_block); let freed_pages = Rc::new(None); - let cb = CodeBlock::new(mem_block.clone(), false, freed_pages.clone()); - let ocb = OutlinedCb::wrap(CodeBlock::new(mem_block, true, freed_pages)); + + let asm_comments = get_option_ref!(dump_disasm).is_some(); + let cb = CodeBlock::new(mem_block.clone(), false, freed_pages.clone(), asm_comments); + let ocb = OutlinedCb::wrap(CodeBlock::new(mem_block, true, freed_pages, asm_comments)); (cb, ocb) }; @@ -10416,9 +11026,9 @@ impl CodegenGlobals { // In test mode we're not linking with the C code // so we don't allocate executable memory #[cfg(test)] - let mut cb = CodeBlock::new_dummy(mem_size / 2); + let mut cb = CodeBlock::new_dummy(exec_mem_size / 2); #[cfg(test)] - let mut ocb = OutlinedCb::wrap(CodeBlock::new_dummy(mem_size / 2)); + let mut ocb = OutlinedCb::wrap(CodeBlock::new_dummy(exec_mem_size / 2)); let ocb_start_addr = ocb.unwrap().get_write_ptr(); let leave_exit_code = gen_leave_exit(&mut ocb).unwrap(); @@ -10433,15 +11043,16 @@ impl CodegenGlobals { let cfunc_exit_code = gen_full_cfunc_return(&mut ocb).unwrap(); let ocb_end_addr = ocb.unwrap().get_write_ptr(); - let ocb_pages = ocb.unwrap().addrs_to_pages(ocb_start_addr, ocb_end_addr); + let ocb_pages = ocb.unwrap().addrs_to_pages(ocb_start_addr, ocb_end_addr).collect(); // Mark all code memory as executable cb.mark_all_executable(); - ocb.unwrap().mark_all_executable(); let codegen_globals = CodegenGlobals { + context_data: BitVector::new(), inline_cb: cb, outlined_cb: ocb, + ocb_pages, leave_exit_code, leave_exception_code, stub_exit_code, @@ -10449,7 +11060,6 @@ impl CodegenGlobals { branch_stub_hit_trampoline, entry_stub_hit_trampoline, global_inval_patches: Vec::new(), - ocb_pages, pc_to_cfunc: HashMap::new(), }; @@ -10468,6 +11078,11 @@ impl CodegenGlobals { unsafe { CODEGEN_GLOBALS.as_mut().is_some() } } + /// Get a mutable reference to the context data + pub fn get_context_data() -> &'static mut BitVector { + &mut CodegenGlobals::get_instance().context_data + } + /// Get a mutable reference to the inline code block pub fn get_inline_cb() -> &'static mut CodeBlock { &mut CodegenGlobals::get_instance().inline_cb @@ -10539,23 +11154,28 @@ impl CodegenGlobals { mod tests { use super::*; - fn setup_codegen() -> (JITState, Context, Assembler, CodeBlock, OutlinedCb) { + fn setup_codegen() -> (Context, Assembler, CodeBlock, OutlinedCb) { let cb = CodeBlock::new_dummy(256 * 1024); return ( - JITState::new( - BlockId { iseq: std::ptr::null(), idx: 0 }, - Context::default(), - cb.get_write_ptr(), - ptr::null(), // No execution context in tests. No peeking! - ), Context::default(), - Assembler::new(), + Assembler::new(0), cb, OutlinedCb::wrap(CodeBlock::new_dummy(256 * 1024)), ); } + fn dummy_jit_state<'a>(cb: &mut CodeBlock, ocb: &'a mut OutlinedCb) -> JITState<'a> { + JITState::new( + BlockId { iseq: std::ptr::null(), idx: 0 }, + Context::default(), + cb.get_write_ptr(), + ptr::null(), // No execution context in tests. No peeking! + ocb, + true, + ) + } + #[test] fn test_gen_leave_exit() { let mut ocb = OutlinedCb::wrap(CodeBlock::new_dummy(256 * 1024)); @@ -10565,7 +11185,7 @@ mod tests { #[test] fn test_gen_exit() { - let (_, _ctx, mut asm, mut cb, _) = setup_codegen(); + let (_ctx, mut asm, mut cb, _) = setup_codegen(); gen_exit(0 as *mut VALUE, &mut asm); asm.compile(&mut cb, None).unwrap(); assert!(cb.get_write_pos() > 0); @@ -10573,7 +11193,7 @@ mod tests { #[test] fn test_get_side_exit() { - let (_jit, ctx, mut asm, _, mut ocb) = setup_codegen(); + let (ctx, mut asm, _, mut ocb) = setup_codegen(); let side_exit_context = SideExitContext::new(0 as _, ctx); asm.get_side_exit(&side_exit_context, None, &mut ocb); assert!(ocb.unwrap().get_write_pos() > 0); @@ -10581,15 +11201,16 @@ mod tests { #[test] fn test_gen_check_ints() { - let (_jit, _ctx, mut asm, _cb, _ocb) = setup_codegen(); + let (_ctx, mut asm, _cb, _ocb) = setup_codegen(); asm.set_side_exit_context(0 as _, 0); gen_check_ints(&mut asm, Counter::guard_send_interrupted); } #[test] fn test_gen_nop() { - let (mut jit, context, mut asm, mut cb, mut ocb) = setup_codegen(); - let status = gen_nop(&mut jit, &mut asm, &mut ocb); + let (context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); + let status = gen_nop(&mut jit, &mut asm); asm.compile(&mut cb, None).unwrap(); assert_eq!(status, Some(KeepCompiling)); @@ -10599,22 +11220,24 @@ mod tests { #[test] fn test_gen_pop() { - let (mut jit, _, mut asm, _cb, mut ocb) = setup_codegen(); + let (_, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); let context = Context::default(); asm.stack_push(Type::Fixnum); - let status = gen_pop(&mut jit, &mut asm, &mut ocb); + let status = gen_pop(&mut jit, &mut asm); assert_eq!(status, Some(KeepCompiling)); let mut default = Context::default(); - default.set_reg_temps(context.get_reg_temps()); + default.set_reg_mapping(context.get_reg_mapping()); assert_eq!(context.diff(&default), TypeDiff::Compatible(0)); } #[test] fn test_gen_dup() { - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); asm.stack_push(Type::Fixnum); - let status = gen_dup(&mut jit, &mut asm, &mut ocb); + let status = gen_dup(&mut jit, &mut asm); assert_eq!(status, Some(KeepCompiling)); @@ -10628,7 +11251,8 @@ mod tests { #[test] fn test_gen_dupn() { - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); asm.stack_push(Type::Fixnum); asm.stack_push(Type::Flonum); @@ -10636,7 +11260,7 @@ mod tests { let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; - let status = gen_dupn(&mut jit, &mut asm, &mut ocb); + let status = gen_dupn(&mut jit, &mut asm); assert_eq!(status, Some(KeepCompiling)); @@ -10651,12 +11275,48 @@ mod tests { } #[test] + fn test_gen_opt_reverse() { + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); + + // Odd number of elements + asm.stack_push(Type::Fixnum); + asm.stack_push(Type::Flonum); + asm.stack_push(Type::CString); + + let mut value_array: [u64; 2] = [0, 3]; + let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; + jit.pc = pc; + + let mut status = gen_opt_reverse(&mut jit, &mut asm); + + assert_eq!(status, Some(KeepCompiling)); + + assert_eq!(Type::CString, asm.ctx.get_opnd_type(StackOpnd(2))); + assert_eq!(Type::Flonum, asm.ctx.get_opnd_type(StackOpnd(1))); + assert_eq!(Type::Fixnum, asm.ctx.get_opnd_type(StackOpnd(0))); + + // Try again with an even number of elements. + asm.stack_push(Type::Nil); + value_array[1] = 4; + status = gen_opt_reverse(&mut jit, &mut asm); + + assert_eq!(status, Some(KeepCompiling)); + + assert_eq!(Type::Nil, asm.ctx.get_opnd_type(StackOpnd(3))); + assert_eq!(Type::Fixnum, asm.ctx.get_opnd_type(StackOpnd(2))); + assert_eq!(Type::Flonum, asm.ctx.get_opnd_type(StackOpnd(1))); + assert_eq!(Type::CString, asm.ctx.get_opnd_type(StackOpnd(0))); + } + + #[test] fn test_gen_swap() { - let (mut jit, _context, mut asm, _cb, mut ocb) = setup_codegen(); + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); asm.stack_push(Type::Fixnum); asm.stack_push(Type::Flonum); - let status = gen_swap(&mut jit, &mut asm, &mut ocb); + let status = gen_swap(&mut jit, &mut asm); let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); let tmp_type_next = asm.ctx.get_opnd_type(StackOpnd(1)); @@ -10668,8 +11328,9 @@ mod tests { #[test] fn test_putnil() { - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); - let status = gen_putnil(&mut jit, &mut asm, &mut ocb); + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); + let status = gen_putnil(&mut jit, &mut asm); let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); @@ -10682,8 +11343,9 @@ mod tests { #[test] fn test_putself() { - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); - let status = gen_putself(&mut jit, &mut asm, &mut ocb); + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); + let status = gen_putself(&mut jit, &mut asm); assert_eq!(status, Some(KeepCompiling)); asm.compile(&mut cb, None).unwrap(); @@ -10692,7 +11354,8 @@ mod tests { #[test] fn test_gen_setn() { - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); asm.stack_push(Type::Fixnum); asm.stack_push(Type::Flonum); asm.stack_push(Type::CString); @@ -10701,7 +11364,7 @@ mod tests { let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; - let status = gen_setn(&mut jit, &mut asm, &mut ocb); + let status = gen_setn(&mut jit, &mut asm); assert_eq!(status, Some(KeepCompiling)); @@ -10715,7 +11378,8 @@ mod tests { #[test] fn test_gen_topn() { - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); asm.stack_push(Type::Flonum); asm.stack_push(Type::CString); @@ -10723,7 +11387,7 @@ mod tests { let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; - let status = gen_topn(&mut jit, &mut asm, &mut ocb); + let status = gen_topn(&mut jit, &mut asm); assert_eq!(status, Some(KeepCompiling)); @@ -10737,7 +11401,8 @@ mod tests { #[test] fn test_gen_adjuststack() { - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); asm.stack_push(Type::Flonum); asm.stack_push(Type::CString); asm.stack_push(Type::Fixnum); @@ -10746,7 +11411,7 @@ mod tests { let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; - let status = gen_adjuststack(&mut jit, &mut asm, &mut ocb); + let status = gen_adjuststack(&mut jit, &mut asm); assert_eq!(status, Some(KeepCompiling)); @@ -10758,10 +11423,11 @@ mod tests { #[test] fn test_gen_leave() { - let (mut jit, _context, mut asm, _cb, mut ocb) = setup_codegen(); + let (_context, mut asm, mut cb, mut ocb) = setup_codegen(); + let mut jit = dummy_jit_state(&mut cb, &mut ocb); // Push return value asm.stack_push(Type::Fixnum); asm.set_side_exit_context(0 as _, 0); - gen_leave(&mut jit, &mut asm, &mut ocb); + gen_leave(&mut jit, &mut asm); } } |