// We use the YARV bytecode constants which have a CRuby-style name #![allow(non_upper_case_globals)] //use crate::asm::x86_64::*; use crate::asm::*; use crate::backend::ir::*; use crate::core::*; use crate::cruby::*; use crate::invariants::*; use crate::options::*; use crate::stats::*; use crate::utils::*; use CodegenStatus::*; use InsnOpnd::*; use std::cell::RefMut; use std::cmp; use std::collections::HashMap; use std::ffi::CStr; use std::mem::{self, size_of}; use std::os::raw::c_uint; use std::ptr; use std::slice; pub use crate::virtualmem::CodePtr; // A block that can be invalidated needs space to write a jump. // We'll reserve a minimum size for any block that could // be invalidated. In this case the JMP takes 5 bytes, but // gen_send_general will always MOV the receiving object // into place, so 2 bytes are always written automatically. //pub const JUMP_SIZE_IN_BYTES: usize = 3; /// Status returned by code generation functions #[derive(PartialEq, Debug)] enum CodegenStatus { KeepCompiling, CantCompile, EndBlock, } /// Code generation function signature type InsnGenFn = fn( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus; /// Code generation state /// This struct only lives while code is being generated pub struct JITState { // Block version being compiled block: BlockRef, // Instruction sequence this is associated with iseq: IseqPtr, // Index of the current instruction being compiled insn_idx: u32, // Opcode for the instruction being compiled opcode: usize, // PC of the instruction being compiled pc: *mut VALUE, // Side exit to the instruction being compiled. See :side-exit:. side_exit_for_pc: Option, // Execution context when compilation started // This allows us to peek at run-time values ec: Option, // Whether we need to record the code address at // the end of this bytecode instruction for global invalidation record_boundary_patch_point: bool, } impl JITState { pub fn new(blockref: &BlockRef) -> Self { JITState { block: blockref.clone(), iseq: ptr::null(), // TODO: initialize this from the blockid insn_idx: 0, opcode: 0, pc: ptr::null_mut::(), side_exit_for_pc: None, ec: None, record_boundary_patch_point: false, } } pub fn get_block(&self) -> BlockRef { self.block.clone() } pub fn get_insn_idx(&self) -> u32 { self.insn_idx } pub fn get_iseq(self: &JITState) -> IseqPtr { self.iseq } pub fn get_opcode(self: &JITState) -> usize { self.opcode } pub fn get_pc(self: &JITState) -> *mut VALUE { self.pc } } use crate::codegen::JCCKinds::*; #[allow(non_camel_case_types, unused)] pub enum JCCKinds { JCC_JNE, JCC_JNZ, JCC_JZ, JCC_JE, JCC_JBE, JCC_JNA, } pub fn jit_get_arg(jit: &JITState, arg_idx: isize) -> VALUE { // insn_len require non-test config #[cfg(not(test))] assert!(insn_len(jit.get_opcode()) > (arg_idx + 1).try_into().unwrap()); unsafe { *(jit.pc.offset(arg_idx + 1)) } } // Get the index of the next instruction fn jit_next_insn_idx(jit: &JITState) -> u32 { jit.insn_idx + insn_len(jit.get_opcode()) } // Check if we are compiling the instruction at the stub PC // Meaning we are compiling the instruction that is next to execute fn jit_at_current_insn(jit: &JITState) -> bool { let ec_pc: *mut VALUE = unsafe { get_cfp_pc(get_ec_cfp(jit.ec.unwrap())) }; ec_pc == jit.pc } // Peek at the nth topmost value on the Ruby stack. // Returns the topmost value when n == 0. fn jit_peek_at_stack(jit: &JITState, ctx: &Context, n: isize) -> VALUE { assert!(jit_at_current_insn(jit)); assert!(n < ctx.get_stack_size() as isize); // Note: this does not account for ctx->sp_offset because // this is only available when hitting a stub, and while // hitting a stub, cfp->sp needs to be up to date in case // codegen functions trigger GC. See :stub-sp-flush:. return unsafe { let sp: *mut VALUE = get_cfp_sp(get_ec_cfp(jit.ec.unwrap())); *(sp.offset(-1 - n)) }; } fn jit_peek_at_self(jit: &JITState) -> VALUE { unsafe { get_cfp_self(get_ec_cfp(jit.ec.unwrap())) } } fn jit_peek_at_local(jit: &JITState, n: i32) -> VALUE { assert!(jit_at_current_insn(jit)); let local_table_size: isize = unsafe { get_iseq_body_local_table_size(jit.iseq) } .try_into() .unwrap(); assert!(n < local_table_size.try_into().unwrap()); unsafe { let ep = get_cfp_ep(get_ec_cfp(jit.ec.unwrap())); let n_isize: isize = n.try_into().unwrap(); let offs: isize = -(VM_ENV_DATA_SIZE as isize) - local_table_size + n_isize + 1; *ep.offset(offs) } } fn jit_peek_at_block_handler(jit: &JITState, level: u32) -> VALUE { assert!(jit_at_current_insn(jit)); unsafe { let ep = get_cfp_ep_level(get_ec_cfp(jit.ec.unwrap()), level); *ep.offset(VM_ENV_DATA_INDEX_SPECVAL as isize) } } // Add a comment at the current position in the code block fn add_comment(cb: &mut CodeBlock, comment_str: &str) { if cfg!(feature = "asm_comments") { cb.add_comment(comment_str); } } /// Increment a profiling counter with counter_name #[cfg(not(feature = "stats"))] macro_rules! gen_counter_incr { ($asm:tt, $counter_name:ident) => {}; } #[cfg(feature = "stats")] macro_rules! gen_counter_incr { ($asm:tt, $counter_name:ident) => { if (get_option!(gen_stats)) { // Get a pointer to the counter variable let ptr = ptr_to_counter!($counter_name); // Load the pointer into a register let ptr_reg = $asm.load(Opnd::const_ptr(ptr as *const u8)); let counter_opnd = Opnd::mem(64, ptr_reg, 0); // Increment and store the updated value $asm.incr_counter(counter_opnd, Opnd::UImm(1)); } }; } /// Increment a counter then take an existing side exit #[cfg(not(feature = "stats"))] macro_rules! counted_exit { ($ocb:tt, $existing_side_exit:tt, $counter_name:ident) => {{ let _ = $ocb; $existing_side_exit }}; } #[cfg(feature = "stats")] macro_rules! counted_exit { ($ocb:tt, $existing_side_exit:tt, $counter_name:ident) => { // The counter is only incremented when stats are enabled if (!get_option!(gen_stats)) { $existing_side_exit } else { let ocb = $ocb.unwrap(); let code_ptr = ocb.get_write_ptr(); let mut ocb_asm = Assembler::new(); // Increment the counter gen_counter_incr!(ocb_asm, $counter_name); // Jump to the existing side exit ocb_asm.jmp($existing_side_exit.into()); ocb_asm.compile(ocb); // Pointer to the side-exit code code_ptr } }; } // Save the incremented PC on the CFP // This is necessary when callees can raise or allocate fn jit_save_pc(jit: &JITState, asm: &mut Assembler) { let pc: *mut VALUE = jit.get_pc(); let ptr: *mut VALUE = unsafe { let cur_insn_len = insn_len(jit.get_opcode()) as isize; pc.offset(cur_insn_len) }; asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(ptr as *const u8)); } /// Save the current SP on the CFP /// This realigns the interpreter SP with the JIT SP /// Note: this will change the current value of REG_SP, /// which could invalidate memory operands fn gen_save_sp(jit: &JITState, asm: &mut Assembler, ctx: &mut Context) { if ctx.get_sp_offset() != 0 { let stack_pointer = ctx.sp_opnd(0); let sp_addr = asm.lea(stack_pointer); asm.mov(SP, sp_addr); let cfp_sp_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP); asm.mov(cfp_sp_opnd, SP); ctx.set_sp_offset(0); } } /// jit_save_pc() + gen_save_sp(). Should be used before calling a routine that /// could: /// - Perform GC allocation /// - Take the VM lock through RB_VM_LOCK_ENTER() /// - Perform Ruby method call fn jit_prepare_routine_call( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler ) { jit.record_boundary_patch_point = true; jit_save_pc(jit, asm); gen_save_sp(jit, asm, ctx); // In case the routine calls Ruby methods, it can set local variables // through Kernel#binding and other means. ctx.clear_local_types(); } /// Record the current codeblock write position for rewriting into a jump into /// the outlined block later. Used to implement global code invalidation. fn record_global_inval_patch(asm: &mut Assembler, outline_block_target_pos: CodePtr) { asm.pos_marker(move |code_ptr| { CodegenGlobals::push_global_inval_patch(code_ptr, outline_block_target_pos); }); } /// Verify the ctx's types and mappings against the compile-time stack, self, /// and locals. fn verify_ctx(jit: &JITState, ctx: &Context) { fn obj_info_str<'a>(val: VALUE) -> &'a str { unsafe { CStr::from_ptr(rb_obj_info(val)).to_str().unwrap() } } // Only able to check types when at current insn assert!(jit_at_current_insn(jit)); let self_val = jit_peek_at_self(jit); let self_val_type = Type::from(self_val); // Verify self operand type if self_val_type.diff(ctx.get_opnd_type(SelfOpnd)) == usize::MAX { panic!( "verify_ctx: ctx self type ({:?}) incompatible with actual value of self {}", ctx.get_opnd_type(SelfOpnd), obj_info_str(self_val) ); } // Verify stack operand types let top_idx = cmp::min(ctx.get_stack_size(), MAX_TEMP_TYPES as u16); for i in 0..top_idx { let (learned_mapping, learned_type) = ctx.get_opnd_mapping(StackOpnd(i)); let stack_val = jit_peek_at_stack(jit, ctx, i as isize); let val_type = Type::from(stack_val); match learned_mapping { TempMapping::MapToSelf => { if self_val != stack_val { panic!( "verify_ctx: stack value was mapped to self, but values did not match!\n stack: {}\n self: {}", obj_info_str(stack_val), obj_info_str(self_val) ); } } TempMapping::MapToLocal(local_idx) => { let local_val = jit_peek_at_local(jit, local_idx.into()); if local_val != stack_val { panic!( "verify_ctx: stack value was mapped to local, but values did not match\n stack: {}\n local {}: {}", obj_info_str(stack_val), local_idx, obj_info_str(local_val) ); } } TempMapping::MapToStack => {} } // If the actual type differs from the learned type if val_type.diff(learned_type) == usize::MAX { panic!( "verify_ctx: ctx type ({:?}) incompatible with actual value on stack: {}", learned_type, obj_info_str(stack_val) ); } } // Verify local variable types let local_table_size = unsafe { get_iseq_body_local_table_size(jit.iseq) }; let top_idx: usize = cmp::min(local_table_size as usize, MAX_TEMP_TYPES); for i in 0..top_idx { let learned_type = ctx.get_local_type(i); let local_val = jit_peek_at_local(jit, i as i32); let local_type = Type::from(local_val); if local_type.diff(learned_type) == usize::MAX { panic!( "verify_ctx: ctx type ({:?}) incompatible with actual value of local: {} (type {:?})", learned_type, obj_info_str(local_val), local_type ); } } } // Fill code_for_exit_from_stub. This is used by branch_stub_hit() to exit // to the interpreter when it cannot service a stub by generating new code. // Before coming here, branch_stub_hit() takes care of fully reconstructing // interpreter state. fn gen_code_for_exit_from_stub(ocb: &mut OutlinedCb) -> CodePtr { let ocb = ocb.unwrap(); let code_ptr = ocb.get_write_ptr(); let mut asm = Assembler::new(); gen_counter_incr!(asm, exit_from_branch_stub); asm.cpop_into(SP); asm.cpop_into(EC); asm.cpop_into(CFP); asm.frame_teardown(); asm.cret(Qundef.into()); asm.compile(ocb); code_ptr } /// Generate an exit to return to the interpreter fn gen_exit(exit_pc: *mut VALUE, ctx: &Context, asm: &mut Assembler) { asm.comment("exit to interpreter"); // Generate the code to exit to the interpreters // Write the adjusted SP back into the CFP if ctx.get_sp_offset() != 0 { let sp_opnd = asm.lea(ctx.sp_opnd(0)); asm.mov( Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), sp_opnd ); } // Update CFP->PC asm.mov( Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(exit_pc as *const u8) ); // Accumulate stats about interpreter exits #[cfg(feature = "stats")] if get_option!(gen_stats) { asm.ccall( rb_yjit_count_side_exit_op as *const u8, vec![Opnd::const_ptr(exit_pc as *const u8)] ); // If --yjit-trace-exits option is enabled, record the exit stack // while recording the side exits. if get_option!(gen_trace_exits) { asm.ccall( rb_yjit_record_exit_stack as *const u8, vec![Opnd::const_ptr(exit_pc as *const u8)] ); } } asm.cpop_into(SP); asm.cpop_into(EC); asm.cpop_into(CFP); asm.frame_teardown(); asm.cret(Qundef.into()); } /// Generate an exit to the interpreter in the outlined code block fn gen_outlined_exit(exit_pc: *mut VALUE, ctx: &Context, ocb: &mut OutlinedCb) -> CodePtr { let mut cb = ocb.unwrap(); let exit_code = cb.get_write_ptr(); let mut asm = Assembler::new(); gen_exit(exit_pc, ctx, &mut asm); asm.compile(&mut cb); exit_code } // :side-exit: // Get an exit for the current instruction in the outlined block. The code // for each instruction often begins with several guards before proceeding // to do work. When guards fail, an option we have is to exit to the // interpreter at an instruction boundary. The piece of code that takes // care of reconstructing interpreter state and exiting out of generated // code is called the side exit. // // No guards change the logic for reconstructing interpreter state at the // moment, so there is one unique side exit for each context. Note that // it's incorrect to jump to the side exit after any ctx stack push/pop operations // since they change the logic required for reconstructing interpreter state. fn get_side_exit(jit: &mut JITState, ocb: &mut OutlinedCb, ctx: &Context) -> CodePtr { match jit.side_exit_for_pc { None => { let exit_code = gen_outlined_exit(jit.pc, ctx, ocb); jit.side_exit_for_pc = Some(exit_code); exit_code } Some(code_ptr) => code_ptr, } } // Ensure that there is an exit for the start of the block being compiled. // Block invalidation uses this exit. pub fn jit_ensure_block_entry_exit(jit: &mut JITState, ocb: &mut OutlinedCb) { let blockref = jit.block.clone(); let mut block = blockref.borrow_mut(); let block_ctx = block.get_ctx(); let blockid = block.get_blockid(); if block.entry_exit.is_some() { return; } // If we're compiling the first instruction in the block. if jit.insn_idx == blockid.idx { // Generate the exit with the cache in jitstate. block.entry_exit = Some(get_side_exit(jit, ocb, &block_ctx)); } else { let pc = unsafe { rb_iseq_pc_at_idx(blockid.iseq, blockid.idx) }; block.entry_exit = Some(gen_outlined_exit(jit.pc, &block_ctx, ocb)); } } // Landing code for when c_return tracing is enabled. See full_cfunc_return(). fn gen_full_cfunc_return(ocb: &mut OutlinedCb) -> CodePtr { let ocb = ocb.unwrap(); let code_ptr = ocb.get_write_ptr(); let mut asm = Assembler::new(); // This chunk of code expects REG_EC to be filled properly and // RAX to contain the return value of the C method. // Call full_cfunc_return() asm.ccall( rb_full_cfunc_return as *const u8, vec![EC, C_RET_OPND] ); // Count the exit gen_counter_incr!(asm, traced_cfunc_return); // Return to the interpreter asm.cpop_into(SP); asm.cpop_into(EC); asm.cpop_into(CFP); asm.frame_teardown(); asm.cret(Qundef.into()); asm.compile(ocb); return code_ptr; } /// Generate a continuation for leave that exits to the interpreter at REG_CFP->pc. /// This is used by gen_leave() and gen_entry_prologue() fn gen_leave_exit(ocb: &mut OutlinedCb) -> CodePtr { let ocb = ocb.unwrap(); let code_ptr = ocb.get_write_ptr(); let mut asm = Assembler::new(); // gen_leave() fully reconstructs interpreter state and leaves the // return value in C_RET_OPND before coming here. let ret_opnd = asm.live_reg_opnd(C_RET_OPND); // Every exit to the interpreter should be counted gen_counter_incr!(asm, leave_interp_return); asm.cpop_into(SP); asm.cpop_into(EC); asm.cpop_into(CFP); asm.frame_teardown(); asm.cret(ret_opnd); asm.compile(ocb); return code_ptr; } // Generate a runtime guard that ensures the PC is at the expected // instruction index in the iseq, otherwise takes a side-exit. // This is to handle the situation of optional parameters. // When a function with optional parameters is called, the entry // PC for the method isn't necessarily 0. fn gen_pc_guard(asm: &mut Assembler, iseq: IseqPtr, insn_idx: u32) { let pc_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC); let expected_pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; let expected_pc_opnd = Opnd::const_ptr(expected_pc as *const u8); asm.cmp(pc_opnd, expected_pc_opnd); let pc_match = asm.new_label("pc_match"); asm.je(pc_match); // We're not starting at the first PC, so we need to exit. gen_counter_incr!(asm, leave_start_pc_non_zero); asm.cpop_into(SP); asm.cpop_into(EC); asm.cpop_into(CFP); asm.frame_teardown(); asm.cret(Qundef.into()); // PC should match the expected insn_idx asm.write_label(pc_match); } /// Compile an interpreter entry block to be inserted into an iseq /// Returns None if compilation fails. pub fn gen_entry_prologue(cb: &mut CodeBlock, iseq: IseqPtr, insn_idx: u32) -> Option { const MAX_PROLOGUE_SIZE: usize = 1024; // Check if we have enough executable memory if !cb.has_capacity(MAX_PROLOGUE_SIZE) { return None; } let old_write_pos = cb.get_write_pos(); // TODO: figure out if this is actually beneficial for performance // Align the current write position to cache line boundaries cb.align_pos(64); let code_ptr = cb.get_write_ptr(); add_comment(cb, "yjit entry"); let mut asm = Assembler::new(); asm.frame_setup(); // Save the CFP, EC, SP registers to the C stack asm.cpush(CFP); asm.cpush(EC); asm.cpush(SP); // We are passed EC and CFP as arguments asm.mov(EC, C_ARG_OPNDS[0]); asm.mov(CFP, C_ARG_OPNDS[1]); // Load the current SP from the CFP into REG_SP asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); // Setup cfp->jit_return asm.mov( Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(CodegenGlobals::get_leave_exit_code().raw_ptr()), ); // We're compiling iseqs that we *expect* to start at `insn_idx`. But in // the case of optional parameters, the interpreter can set the pc to a // different location depending on the optional parameters. If an iseq // has optional parameters, we'll add a runtime check that the PC we've // compiled for is the same PC that the interpreter wants us to run with. // If they don't match, then we'll take a side exit. if unsafe { get_iseq_flags_has_opt(iseq) } { gen_pc_guard(&mut asm, iseq, insn_idx); } asm.compile(cb); // Verify MAX_PROLOGUE_SIZE assert!(cb.get_write_pos() - old_write_pos <= MAX_PROLOGUE_SIZE); return Some(code_ptr); } // Generate code to check for interrupts and take a side-exit. // Warning: this function clobbers REG0 fn gen_check_ints(asm: &mut Assembler, side_exit: CodePtr) { // Check for interrupts // see RUBY_VM_CHECK_INTS(ec) macro asm.comment("RUBY_VM_CHECK_INTS(ec)"); let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK)); asm.test( Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG), not_mask, ); asm.jnz(Target::CodePtr(side_exit)); } // Generate a stubbed unconditional jump to the next bytecode instruction. // Blocks that are part of a guard chain can use this to share the same successor. fn jump_to_next_insn( jit: &mut JITState, current_context: &Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) { // Reset the depth since in current usages we only ever jump to to // chain_depth > 0 from the same instruction. let mut reset_depth = *current_context; reset_depth.reset_chain_depth(); let jump_block = BlockId { iseq: jit.iseq, idx: jit_next_insn_idx(jit), }; // We are at the end of the current instruction. Record the boundary. if jit.record_boundary_patch_point { let exit_pc = unsafe { jit.pc.offset(insn_len(jit.opcode).try_into().unwrap()) }; let exit_pos = gen_outlined_exit(exit_pc, &reset_depth, ocb); record_global_inval_patch(asm, exit_pos); jit.record_boundary_patch_point = false; } // Generate the jump instruction gen_direct_jump(jit, &reset_depth, jump_block, asm); } // Compile a sequence of bytecode instructions for a given basic block version. // Part of gen_block_version(). // Note: this function will mutate its context while generating code, // but the input start_ctx argument should remain immutable. pub fn gen_single_block( blockid: BlockId, start_ctx: &Context, ec: EcPtr, cb: &mut CodeBlock, ocb: &mut OutlinedCb, ) -> Result { // Limit the number of specialized versions for this block let mut ctx = limit_block_versions(blockid, start_ctx); verify_blockid(blockid); assert!(!(blockid.idx == 0 && ctx.get_stack_size() > 0)); // Instruction sequence to compile let iseq = blockid.iseq; let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; let mut insn_idx: c_uint = blockid.idx; let starting_insn_idx = insn_idx; // Allocate the new block let blockref = Block::new(blockid, &ctx); // Initialize a JIT state object let mut jit = JITState::new(&blockref); jit.iseq = blockid.iseq; jit.ec = Some(ec); // Mark the start position of the block blockref.borrow_mut().set_start_addr(cb.get_write_ptr()); // Create a backend assembler instance let mut asm = Assembler::new(); // For each instruction to compile // NOTE: could rewrite this loop with a std::iter::Iterator while insn_idx < iseq_size { // Get the current pc and opcode let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. let opcode: usize = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } .try_into() .unwrap(); // opt_getinlinecache wants to be in a block all on its own. Cut the block short // if we run into it. See gen_opt_getinlinecache() for details. if opcode == YARVINSN_opt_getinlinecache.as_usize() && insn_idx > starting_insn_idx { jump_to_next_insn(&mut jit, &ctx, &mut asm, ocb); break; } // Set the current instruction jit.insn_idx = insn_idx; jit.opcode = opcode; jit.pc = pc; jit.side_exit_for_pc = None; // If previous instruction requested to record the boundary if jit.record_boundary_patch_point { // Generate an exit to this instruction and record it let exit_pos = gen_outlined_exit(jit.pc, &ctx, ocb); record_global_inval_patch(&mut asm, exit_pos); jit.record_boundary_patch_point = false; } // In debug mode, verify our existing assumption if cfg!(debug_assertions) && get_option!(verify_ctx) && jit_at_current_insn(&jit) { verify_ctx(&jit, &ctx); } // Lookup the codegen function for this instruction let mut status = CantCompile; if let Some(gen_fn) = get_gen_fn(VALUE(opcode)) { // :count-placement: // Count bytecode instructions that execute in generated code. // Note that the increment happens even when the output takes side exit. gen_counter_incr!(asm, exec_instruction); // Add a comment for the name of the YARV instruction asm.comment(&insn_name(opcode)); // If requested, dump instructions for debugging if get_option!(dump_insns) { println!("compiling {}", insn_name(opcode)); print_str(&mut asm, &format!("executing {}", insn_name(opcode))); } // Call the code generation function status = gen_fn(&mut jit, &mut ctx, &mut asm, ocb); } // If we can't compile this instruction // exit to the interpreter and stop compiling if status == CantCompile { if get_option!(dump_insns) { println!("can't compile {}", insn_name(opcode)); } let mut block = jit.block.borrow_mut(); // TODO: if the codegen function makes changes to ctx and then return YJIT_CANT_COMPILE, // the exit this generates would be wrong. We could save a copy of the entry context // and assert that ctx is the same here. gen_exit(jit.pc, &ctx, &mut asm); // If this is the first instruction in the block, then we can use // the exit for block->entry_exit. if insn_idx == block.get_blockid().idx { block.entry_exit = block.get_start_addr(); } break; } // For now, reset the chain depth after each instruction as only the // first instruction in the block can concern itself with the depth. ctx.reset_chain_depth(); // Move to the next instruction to compile insn_idx += insn_len(opcode); // If the instruction terminates this block if status == EndBlock { break; } } // Finish filling out the block { // Compile code into the code block let gc_offsets = asm.compile(cb); let mut block = jit.block.borrow_mut(); // Add the GC offsets to the block for offset in gc_offsets { block.add_gc_obj_offset(offset) } // Mark the end position of the block block.set_end_addr(cb.get_write_ptr()); // Store the index of the last instruction in the block block.set_end_idx(insn_idx); } // We currently can't handle cases where the request is for a block that // doesn't go to the next instruction. assert!(!jit.record_boundary_patch_point); // If code for the block doesn't fit, fail if cb.has_dropped_bytes() || ocb.unwrap().has_dropped_bytes() { return Err(()); } // Block compiled successfully Ok(blockref) } fn gen_nop( _jit: &mut JITState, _ctx: &mut Context, _asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // Do nothing KeepCompiling } fn gen_pop( _jit: &mut JITState, ctx: &mut Context, _asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // Decrement SP ctx.stack_pop(1); KeepCompiling } fn gen_dup( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let dup_val = ctx.stack_pop(0); let (mapping, tmp_type) = ctx.get_opnd_mapping(StackOpnd(0)); let loc0 = ctx.stack_push_mapping((mapping, tmp_type)); asm.mov(loc0, dup_val); KeepCompiling } // duplicate stack top n elements fn gen_dupn( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let nval: VALUE = jit_get_arg(jit, 0); let VALUE(n) = nval; // In practice, seems to be only used for n==2 if n != 2 { return CantCompile; } let opnd1: Opnd = ctx.stack_opnd(1); let opnd0: Opnd = ctx.stack_opnd(0); let mapping1 = ctx.get_opnd_mapping(StackOpnd(1)); let mapping0 = ctx.get_opnd_mapping(StackOpnd(0)); let dst1: Opnd = ctx.stack_push_mapping(mapping1); asm.mov(dst1, opnd1); let dst0: Opnd = ctx.stack_push_mapping(mapping0); asm.mov(dst0, opnd0); KeepCompiling } // Swap top 2 stack entries fn gen_swap( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { stack_swap(jit, ctx, asm, 0, 1); KeepCompiling } fn stack_swap( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, offset0: u16, offset1: u16, ) { let stack0_mem = ctx.stack_opnd(offset0 as i32); let stack1_mem = ctx.stack_opnd(offset1 as i32); let mapping0 = ctx.get_opnd_mapping(StackOpnd(offset0)); let mapping1 = ctx.get_opnd_mapping(StackOpnd(offset1)); let stack0_reg = asm.load(stack0_mem); let stack1_reg = asm.load(stack1_mem); asm.mov(stack0_mem, stack1_reg); asm.mov(stack1_mem, stack0_reg); ctx.set_opnd_mapping(StackOpnd(offset0), mapping1); ctx.set_opnd_mapping(StackOpnd(offset1), mapping0); } fn gen_putnil( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { jit_putobject(jit, ctx, asm, Qnil); KeepCompiling } fn jit_putobject(jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, arg: VALUE) { let val_type: Type = Type::from(arg); let stack_top = ctx.stack_push(val_type); asm.mov(stack_top, arg.into()); } fn gen_putobject_int2fix( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let opcode = jit.opcode; let cst_val: usize = if opcode == YARVINSN_putobject_INT2FIX_0_.as_usize() { 0 } else { 1 }; jit_putobject(jit, ctx, asm, VALUE::fixnum_from_usize(cst_val)); KeepCompiling } fn gen_putobject( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let arg: VALUE = jit_get_arg(jit, 0); jit_putobject(jit, ctx, asm, arg); KeepCompiling } fn gen_putself( _jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // Write it on the stack let stack_top = ctx.stack_push_self(); asm.mov( stack_top, Opnd::mem((8 * SIZEOF_VALUE) as u8, CFP, RUBY_OFFSET_CFP_SELF) ); KeepCompiling } fn gen_putspecialobject( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let object_type = jit_get_arg(jit, 0); if object_type == VALUE(VM_SPECIAL_OBJECT_VMCORE.as_usize()) { let stack_top = ctx.stack_push(Type::UnknownHeap); let frozen_core = unsafe { rb_mRubyVMFrozenCore }; asm.mov(stack_top, frozen_core.into()); KeepCompiling } else { // TODO: implement for VM_SPECIAL_OBJECT_CBASE and // VM_SPECIAL_OBJECT_CONST_BASE CantCompile } } // set Nth stack entry to stack top fn gen_setn( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let n: VALUE = jit_get_arg(jit, 0); let top_val = ctx.stack_pop(0); let dst_opnd = ctx.stack_opnd(n.into()); asm.mov( dst_opnd, top_val ); let mapping = ctx.get_opnd_mapping(StackOpnd(0)); ctx.set_opnd_mapping(StackOpnd(n.into()), mapping); KeepCompiling } // get nth stack value, then push it fn gen_topn( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let nval = jit_get_arg(jit, 0); let top_n_val = ctx.stack_opnd(nval.into()); let mapping = ctx.get_opnd_mapping(StackOpnd(nval.into())); let loc0 = ctx.stack_push_mapping(mapping); asm.mov(loc0, top_n_val); KeepCompiling } // Pop n values off the stack fn gen_adjuststack( jit: &mut JITState, ctx: &mut Context, _cb: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let nval: VALUE = jit_get_arg(jit, 0); let VALUE(n) = nval; ctx.stack_pop(n); KeepCompiling } fn gen_opt_plus( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let comptime_a = jit_peek_at_stack(jit, ctx, 1); let comptime_b = jit_peek_at_stack(jit, ctx, 0); if comptime_a.fixnum_p() && comptime_b.fixnum_p() { // Create a side-exit to fall back to the interpreter // Note: we generate the side-exit before popping operands from the stack let side_exit = get_side_exit(jit, ocb, ctx); if !assume_bop_not_redefined(jit, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) { return CantCompile; } // Check that both operands are fixnums guard_two_fixnums(ctx, asm, side_exit); // Get the operands and destination from the stack let arg1 = ctx.stack_pop(1); let arg0 = ctx.stack_pop(1); // Add arg0 + arg1 and test for overflow let arg0_untag = asm.sub(arg0, Opnd::Imm(1)); let out_val = asm.add(arg0_untag, arg1); asm.jo(side_exit.into()); // Push the output on the stack let dst = ctx.stack_push(Type::Fixnum); asm.mov(dst, out_val); KeepCompiling } else { gen_opt_send_without_block(jit, ctx, asm, ocb) } } // new array initialized from top N values fn gen_newarray( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let n = jit_get_arg(jit, 0).as_u32(); // Save the PC and SP because we are allocating jit_prepare_routine_call(jit, ctx, asm); let offset_magnitude = SIZEOF_VALUE as u32 * n; let values_opnd = ctx.sp_opnd(-(offset_magnitude as isize)); let values_ptr = asm.lea(values_opnd); // call rb_ec_ary_new_from_values(struct rb_execution_context_struct *ec, long n, const VALUE *elts); let new_ary = asm.ccall( rb_ec_ary_new_from_values as *const u8, vec![ EC, Opnd::UImm(n.into()), values_ptr ] ); ctx.stack_pop(n.as_usize()); let stack_ret = ctx.stack_push(Type::Array); asm.mov(stack_ret, new_ary); KeepCompiling } // dup array fn gen_duparray( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let ary = jit_get_arg(jit, 0); // Save the PC and SP because we are allocating jit_prepare_routine_call(jit, ctx, asm); // call rb_ary_resurrect(VALUE ary); let new_ary = asm.ccall( rb_ary_resurrect as *const u8, vec![ary.into()], ); let stack_ret = ctx.stack_push(Type::Array); asm.mov(stack_ret, new_ary); KeepCompiling } // dup hash fn gen_duphash( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let hash = jit_get_arg(jit, 0); // Save the PC and SP because we are allocating jit_prepare_routine_call(jit, ctx, asm); // call rb_hash_resurrect(VALUE hash); let hash = asm.ccall(rb_hash_resurrect as *const u8, vec![hash.into()]); let stack_ret = ctx.stack_push(Type::Hash); asm.mov(stack_ret, hash); KeepCompiling } // call to_a on the array on the stack fn gen_splatarray( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let flag = jit_get_arg(jit, 0); // Save the PC and SP because the callee may allocate // Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm); // Get the operands from the stack let ary_opnd = ctx.stack_pop(1); // Call rb_vm_splat_array(flag, ary) let ary = asm.ccall(rb_vm_splat_array as *const u8, vec![flag.into(), ary_opnd]); let stack_ret = ctx.stack_push(Type::Array); asm.mov(stack_ret, ary); KeepCompiling } // concat two arrays fn gen_concatarray( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // Save the PC and SP because the callee may allocate // Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm); // Get the operands from the stack let ary2st_opnd = ctx.stack_pop(1); let ary1_opnd = ctx.stack_pop(1); // Call rb_vm_concat_array(ary1, ary2st) let ary = asm.ccall(rb_vm_concat_array as *const u8, vec![ary1_opnd, ary2st_opnd]); let stack_ret = ctx.stack_push(Type::Array); asm.mov(stack_ret, ary); KeepCompiling } // new range initialized from top 2 values fn gen_newrange( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let flag = jit_get_arg(jit, 0); // rb_range_new() allocates and can raise jit_prepare_routine_call(jit, ctx, asm); // val = rb_range_new(low, high, (int)flag); let range_opnd = asm.ccall( rb_range_new as *const u8, vec![ ctx.stack_opnd(1), ctx.stack_opnd(0), flag.into() ] ); ctx.stack_pop(2); let stack_ret = ctx.stack_push(Type::UnknownHeap); asm.mov(stack_ret, range_opnd); KeepCompiling } fn guard_object_is_heap( asm: &mut Assembler, object_opnd: Opnd, side_exit: CodePtr, ) { asm.comment("guard object is heap"); // Test that the object is not an immediate asm.test(object_opnd, (RUBY_IMMEDIATE_MASK as u64).into()); asm.jnz(side_exit.into()); // Test that the object is not false or nil asm.cmp(object_opnd, Qnil.into()); asm.jbe(side_exit.into()); } fn guard_object_is_array( asm: &mut Assembler, object_opnd: Opnd, side_exit: CodePtr, ) { asm.comment("guard object is array"); // Pull out the type mask let flags_opnd = Opnd::mem( 8 * SIZEOF_VALUE as u8, object_opnd, RUBY_OFFSET_RBASIC_FLAGS, ); let flags_opnd = asm.and(flags_opnd, (RUBY_T_MASK as u64).into()); // Compare the result with T_ARRAY asm.cmp(flags_opnd, (RUBY_T_ARRAY as u64).into()); asm.jne(side_exit.into()); } /* fn guard_object_is_string( cb: &mut CodeBlock, object_reg: X86Opnd, flags_reg: X86Opnd, side_exit: CodePtr, ) { add_comment(cb, "guard object is string"); // Pull out the type mask mov( cb, flags_reg, mem_opnd( 8 * SIZEOF_VALUE as u8, object_reg, RUBY_OFFSET_RBASIC_FLAGS, ), ); and(cb, flags_reg, uimm_opnd(RUBY_T_MASK as u64)); // Compare the result with T_STRING cmp(cb, flags_reg, uimm_opnd(RUBY_T_STRING as u64)); jne_ptr(cb, side_exit); } */ // push enough nils onto the stack to fill out an array fn gen_expandarray( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let flag = jit_get_arg(jit, 1); let VALUE(flag_value) = flag; // If this instruction has the splat flag, then bail out. if flag_value & 0x01 != 0 { gen_counter_incr!(asm, expandarray_splat); return CantCompile; } // If this instruction has the postarg flag, then bail out. if flag_value & 0x02 != 0 { gen_counter_incr!(asm, expandarray_postarg); return CantCompile; } let side_exit = get_side_exit(jit, ocb, ctx); // num is the number of requested values. If there aren't enough in the // array then we're going to push on nils. let num = jit_get_arg(jit, 0); let array_type = ctx.get_opnd_type(StackOpnd(0)); let array_opnd = ctx.stack_pop(1); if matches!(array_type, Type::Nil) { // special case for a, b = nil pattern // push N nils onto the stack for _i in 0..(num.into()) { let push_opnd = ctx.stack_push(Type::Nil); asm.mov(push_opnd, Qnil.into()); } return KeepCompiling; } // Move the array from the stack into REG0 and check that it's an array. let array_reg = asm.load(array_opnd); guard_object_is_heap( asm, array_reg, counted_exit!(ocb, side_exit, expandarray_not_array), ); guard_object_is_array( asm, array_reg, counted_exit!(ocb, side_exit, expandarray_not_array), ); // If we don't actually want any values, then just return. if num == VALUE(0) { return KeepCompiling; } // Pull out the embed flag to check if it's an embedded array. let flags_opnd = Opnd::mem((8 * SIZEOF_VALUE) as u8, array_reg, RUBY_OFFSET_RBASIC_FLAGS); // Move the length of the embedded array into REG1. let emb_len_opnd = asm.and(flags_opnd, (RARRAY_EMBED_LEN_MASK as u64).into()); let emb_len_opnd = asm.rshift(emb_len_opnd, (RARRAY_EMBED_LEN_SHIFT as u64).into()); // Conditionally move the length of the heap array into REG1. let flags_opnd = Opnd::mem((8 * SIZEOF_VALUE) as u8, array_reg, RUBY_OFFSET_RBASIC_FLAGS); asm.test(flags_opnd, (RARRAY_EMBED_FLAG as u64).into()); let array_len_opnd = Opnd::mem( (8 * size_of::()) as u8, asm.load(array_opnd), RUBY_OFFSET_RARRAY_AS_HEAP_LEN, ); let array_len_opnd = asm.csel_nz(emb_len_opnd, array_len_opnd); // Only handle the case where the number of values in the array is greater // than or equal to the number of values requested. asm.cmp(array_len_opnd, num.into()); asm.jo(counted_exit!(ocb, side_exit, expandarray_rhs_too_small).into()); // Load the address of the embedded array into REG1. // (struct RArray *)(obj)->as.ary let array_reg = asm.load(array_opnd); let ary_opnd = asm.lea(Opnd::mem((8 * SIZEOF_VALUE) as u8, array_reg, RUBY_OFFSET_RARRAY_AS_ARY)); // Conditionally load the address of the heap array into REG1. // (struct RArray *)(obj)->as.heap.ptr let flags_opnd = Opnd::mem((8 * SIZEOF_VALUE) as u8, array_reg, RUBY_OFFSET_RBASIC_FLAGS); asm.test(flags_opnd, Opnd::UImm(RARRAY_EMBED_FLAG as u64)); let heap_ptr_opnd = Opnd::mem( (8 * size_of::()) as u8, asm.load(array_opnd), RUBY_OFFSET_RARRAY_AS_HEAP_PTR, ); let ary_opnd = asm.csel_nz(ary_opnd, heap_ptr_opnd); // Loop backward through the array and push each element onto the stack. for i in (0..(num.as_i32())).rev() { let top = ctx.stack_push(Type::Unknown); asm.mov(top, Opnd::mem(64, ary_opnd, i * (SIZEOF_VALUE as i32))); } KeepCompiling } fn gen_getlocal_wc0( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // Compute the offset from BP to the local let slot_idx = jit_get_arg(jit, 0).as_i32(); let offs: i32 = -(SIZEOF_VALUE as i32) * slot_idx; let local_idx = slot_to_local_idx(jit.get_iseq(), slot_idx); // Load environment pointer EP (level 0) from CFP let ep_opnd = gen_get_ep(asm, 0); // Load the local from the EP let local_opnd = Opnd::mem(64, ep_opnd, offs); // Write the local at SP let stack_top = ctx.stack_push_local(local_idx.as_usize()); asm.mov(stack_top, local_opnd); KeepCompiling } // Compute the index of a local variable from its slot index fn slot_to_local_idx(iseq: IseqPtr, slot_idx: i32) -> u32 { // Layout illustration // This is an array of VALUE // | VM_ENV_DATA_SIZE | // v v // low addr <+-------+-------+-------+-------+------------------+ // |local 0|local 1| ... |local n| .... | // +-------+-------+-------+-------+------------------+ // ^ ^ ^ ^ // +-------+---local_table_size----+ cfp->ep--+ // | | // +------------------slot_idx----------------+ // // See usages of local_var_name() from iseq.c for similar calculation. // Equivalent of iseq->body->local_table_size let local_table_size: i32 = unsafe { get_iseq_body_local_table_size(iseq) } .try_into() .unwrap(); let op = slot_idx - (VM_ENV_DATA_SIZE as i32); let local_idx = local_table_size - op - 1; assert!(local_idx >= 0 && local_idx < local_table_size); local_idx.try_into().unwrap() } // Get EP at level from CFP fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { // Load environment pointer EP from CFP into a register let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); let mut ep_opnd = asm.load(ep_opnd); for _ in (0..level).rev() { // Get the previous EP from the current EP // See GET_PREV_EP(ep) macro // VALUE *prev_ep = ((VALUE *)((ep)[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03)) let offs = (SIZEOF_VALUE as i32) * (VM_ENV_DATA_INDEX_SPECVAL as i32); ep_opnd = asm.load(Opnd::mem(64, ep_opnd, offs)); ep_opnd = asm.and(ep_opnd, Opnd::Imm(!0x03)); } ep_opnd } fn gen_getlocal_generic( ctx: &mut Context, asm: &mut Assembler, local_idx: u32, level: u32, ) -> CodegenStatus { // Load environment pointer EP (level 0) from CFP let ep_opnd = gen_get_ep(asm, level); // Load the local from the block // val = *(vm_get_ep(GET_EP(), level) - idx); let offs = -(SIZEOF_VALUE as i32 * local_idx as i32); let local_opnd = Opnd::mem(64, ep_opnd, offs); // Write the local at SP let stack_top = ctx.stack_push(Type::Unknown); asm.mov(stack_top, local_opnd); KeepCompiling } fn gen_getlocal( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let idx = jit_get_arg(jit, 0); let level = jit_get_arg(jit, 1); gen_getlocal_generic(ctx, asm, idx.as_u32(), level.as_u32()) } fn gen_getlocal_wc1( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let idx = jit_get_arg(jit, 0); gen_getlocal_generic(ctx, asm, idx.as_u32(), 1) } fn gen_setlocal_wc0( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { /* vm_env_write(const VALUE *ep, int index, VALUE v) { VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS]; if (LIKELY((flags & VM_ENV_FLAG_WB_REQUIRED) == 0)) { VM_STACK_ENV_WRITE(ep, index, v); } else { vm_env_write_slowpath(ep, index, v); } } */ let slot_idx = jit_get_arg(jit, 0).as_i32(); let local_idx = slot_to_local_idx(jit.get_iseq(), slot_idx).as_usize(); let value_type = ctx.get_opnd_type(StackOpnd(0)); // Load environment pointer EP (level 0) from CFP let ep_opnd = gen_get_ep(asm, 0); // Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers // only affect heap objects being written. If we know an immediate value is being written we // can skip this check. if !value_type.is_imm() { // flags & VM_ENV_FLAG_WB_REQUIRED let flags_opnd = Opnd::mem( 64, ep_opnd, SIZEOF_VALUE as i32 * VM_ENV_DATA_INDEX_FLAGS as i32, ); asm.test(flags_opnd, VM_ENV_FLAG_WB_REQUIRED.into()); // Create a side-exit to fall back to the interpreter let side_exit = get_side_exit(jit, ocb, ctx); // if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0 asm.jnz(side_exit.into()); } // Set the type of the local variable in the context ctx.set_local_type(local_idx, value_type); // Pop the value to write from the stack let stack_top = ctx.stack_pop(1); // Write the value at the environment pointer let offs: i32 = -8 * slot_idx; asm.mov(Opnd::mem(64, ep_opnd, offs), stack_top); KeepCompiling } fn gen_setlocal_generic( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, local_idx: i32, level: u32, ) -> CodegenStatus { let value_type = ctx.get_opnd_type(StackOpnd(0)); // Load environment pointer EP at level let ep_opnd = gen_get_ep(asm, level); // Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers // only affect heap objects being written. If we know an immediate value is being written we // can skip this check. if !value_type.is_imm() { // flags & VM_ENV_FLAG_WB_REQUIRED let flags_opnd = Opnd::mem( 64, ep_opnd, SIZEOF_VALUE as i32 * VM_ENV_DATA_INDEX_FLAGS as i32, ); asm.test(flags_opnd, VM_ENV_FLAG_WB_REQUIRED.into()); // Create a side-exit to fall back to the interpreter let side_exit = get_side_exit(jit, ocb, ctx); // if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0 asm.jnz(side_exit.into()); } // Pop the value to write from the stack let stack_top = ctx.stack_pop(1); // Write the value at the environment pointer let offs = -(SIZEOF_VALUE as i32 * local_idx); asm.mov(Opnd::mem(64, ep_opnd, offs), stack_top); KeepCompiling } fn gen_setlocal( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let idx = jit_get_arg(jit, 0).as_i32(); let level = jit_get_arg(jit, 1).as_u32(); gen_setlocal_generic(jit, ctx, asm, ocb, idx, level) } fn gen_setlocal_wc1( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let idx = jit_get_arg(jit, 0).as_i32(); gen_setlocal_generic(jit, ctx, asm, ocb, idx, 1) } // new hash initialized from top N values fn gen_newhash( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let num: u64 = jit_get_arg(jit, 0).as_u64(); // Save the PC and SP because we are allocating jit_prepare_routine_call(jit, ctx, asm); if num != 0 { // val = rb_hash_new_with_size(num / 2); let new_hash = asm.ccall( rb_hash_new_with_size as *const u8, vec![Opnd::UImm(num / 2)] ); // Save the allocated hash as we want to push it after insertion asm.cpush(new_hash); asm.cpush(new_hash); // x86 alignment // Get a pointer to the values to insert into the hash let stack_addr_from_top = asm.lea(ctx.stack_opnd((num - 1) as i32)); // rb_hash_bulk_insert(num, STACK_ADDR_FROM_TOP(num), val); asm.ccall( rb_hash_bulk_insert as *const u8, vec![ Opnd::UImm(num), stack_addr_from_top, new_hash ] ); let new_hash = asm.cpop(); asm.cpop_into(new_hash); // x86 alignment ctx.stack_pop(num.try_into().unwrap()); let stack_ret = ctx.stack_push(Type::Hash); asm.mov(stack_ret, new_hash); } else { // val = rb_hash_new(); let new_hash = asm.ccall(rb_hash_new as *const u8, vec![]); let stack_ret = ctx.stack_push(Type::Hash); asm.mov(stack_ret, new_hash); } KeepCompiling } fn gen_putstring( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let put_val = jit_get_arg(jit, 0); // Save the PC and SP because the callee will allocate jit_prepare_routine_call(jit, ctx, asm); let str_opnd = asm.ccall( rb_ec_str_resurrect as *const u8, vec![EC, put_val.into()] ); let stack_top = ctx.stack_push(Type::CString); asm.mov(stack_top, str_opnd); KeepCompiling } // Push Qtrue or Qfalse depending on whether the given keyword was supplied by // the caller fn gen_checkkeyword( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // When a keyword is unspecified past index 32, a hash will be used // instead. This can only happen in iseqs taking more than 32 keywords. if unsafe { (*get_iseq_body_param_keyword(jit.iseq)).num >= 32 } { return CantCompile; } // The EP offset to the undefined bits local let bits_offset = jit_get_arg(jit, 0).as_i32(); // The index of the keyword we want to check let index: i64 = jit_get_arg(jit, 1).as_i64(); // Load environment pointer EP let ep_opnd = gen_get_ep(asm, 0); // VALUE kw_bits = *(ep - bits); let bits_opnd = Opnd::mem(64, ep_opnd, (SIZEOF_VALUE as i32) * -bits_offset); // unsigned int b = (unsigned int)FIX2ULONG(kw_bits); // if ((b & (0x01 << idx))) { // // We can skip the FIX2ULONG conversion by shifting the bit we test let bit_test: i64 = 0x01 << (index + 1); asm.test(bits_opnd, Opnd::Imm(bit_test)); let ret_opnd = asm.csel_z(Qtrue.into(), Qfalse.into()); let stack_ret = ctx.stack_push(Type::UnknownImm); asm.mov(stack_ret, ret_opnd); KeepCompiling } fn gen_jnz_to_target0( asm: &mut Assembler, target0: CodePtr, _target1: Option, shape: BranchShape, ) { match shape { BranchShape::Next0 | BranchShape::Next1 => unreachable!(), BranchShape::Default => asm.jnz(target0.into()), } } fn gen_jz_to_target0( asm: &mut Assembler, target0: CodePtr, _target1: Option, shape: BranchShape, ) { match shape { BranchShape::Next0 | BranchShape::Next1 => unreachable!(), BranchShape::Default => asm.jz(Target::CodePtr(target0)), } } fn gen_jbe_to_target0( asm: &mut Assembler, target0: CodePtr, _target1: Option, shape: BranchShape, ) { match shape { BranchShape::Next0 | BranchShape::Next1 => unreachable!(), BranchShape::Default => asm.jbe(Target::CodePtr(target0)), } } // Generate a jump to a stub that recompiles the current YARV instruction on failure. // When depth_limit is exceeded, generate a jump to a side exit. fn jit_chain_guard( jcc: JCCKinds, jit: &JITState, ctx: &Context, asm: &mut Assembler, ocb: &mut OutlinedCb, depth_limit: i32, side_exit: CodePtr, ) { let target0_gen_fn = match jcc { JCC_JNE | JCC_JNZ => gen_jnz_to_target0, JCC_JZ | JCC_JE => gen_jz_to_target0, JCC_JBE | JCC_JNA => gen_jbe_to_target0, }; if (ctx.get_chain_depth() as i32) < depth_limit { let mut deeper = *ctx; deeper.increment_chain_depth(); let bid = BlockId { iseq: jit.iseq, idx: jit.insn_idx, }; gen_branch(jit, ctx, asm, ocb, bid, &deeper, None, None, target0_gen_fn); } else { target0_gen_fn(asm, side_exit, None, BranchShape::Default); } } // up to 5 different classes, and embedded or not for each pub const GET_IVAR_MAX_DEPTH: i32 = 10; // hashes and arrays pub const OPT_AREF_MAX_CHAIN_DEPTH: i32 = 2; // up to 5 different classes pub const SEND_MAX_DEPTH: i32 = 5; // Codegen for setting an instance variable. // Preconditions: // - receiver is in REG0 // - receiver has the same class as CLASS_OF(comptime_receiver) // - no stack push or pops to ctx since the entry to the codegen of the instruction being compiled fn gen_set_ivar( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, recv: VALUE, ivar_name: ID, ) -> CodegenStatus { // Save the PC and SP because the callee may allocate // Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm); // Get the operands from the stack let val_opnd = ctx.stack_pop(1); let recv_opnd = ctx.stack_pop(1); let ivar_index: u32 = unsafe { rb_obj_ensure_iv_index_mapping(recv, ivar_name) }; // Call rb_vm_set_ivar_idx with the receiver, the index of the ivar, and the value let val = asm.ccall( rb_vm_set_ivar_idx as *const u8, vec![ recv_opnd, Opnd::Imm(ivar_index.into()), val_opnd, ], ); let out_opnd = ctx.stack_push(Type::Unknown); asm.mov(out_opnd, val); KeepCompiling } // Codegen for getting an instance variable. // Preconditions: // - receiver has the same class as CLASS_OF(comptime_receiver) // - no stack push or pops to ctx since the entry to the codegen of the instruction being compiled fn gen_get_ivar( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, max_chain_depth: i32, comptime_receiver: VALUE, ivar_name: ID, recv: Opnd, recv_opnd: InsnOpnd, side_exit: CodePtr, ) -> CodegenStatus { let comptime_val_klass = comptime_receiver.class_of(); let starting_context = *ctx; // make a copy for use with jit_chain_guard // If recv isn't already a register, load it. let recv = match recv { Opnd::Reg(_) => recv, _ => asm.load(recv), }; // Check if the comptime class uses a custom allocator let custom_allocator = unsafe { rb_get_alloc_func(comptime_val_klass) }; let uses_custom_allocator = match custom_allocator { Some(alloc_fun) => { let allocate_instance = rb_class_allocate_instance as *const u8; alloc_fun as *const u8 != allocate_instance } None => false, }; // Check if the comptime receiver is a T_OBJECT let receiver_t_object = unsafe { RB_TYPE_P(comptime_receiver, RUBY_T_OBJECT) }; // If the class uses the default allocator, instances should all be T_OBJECT // NOTE: This assumes nobody changes the allocator of the class after allocation. // Eventually, we can encode whether an object is T_OBJECT or not // inside object shapes. if !receiver_t_object || uses_custom_allocator { // General case. Call rb_ivar_get(). // VALUE rb_ivar_get(VALUE obj, ID id) asm.comment("call rb_ivar_get()"); // The function could raise exceptions. jit_prepare_routine_call(jit, ctx, asm); let ivar_val = asm.ccall(rb_ivar_get as *const u8, vec![recv, Opnd::UImm(ivar_name)]); if recv_opnd != SelfOpnd { ctx.stack_pop(1); } // Push the ivar on the stack let out_opnd = ctx.stack_push(Type::Unknown); asm.mov(out_opnd, ivar_val); // Jump to next instruction. This allows guard chains to share the same successor. jump_to_next_insn(jit, ctx, asm, ocb); return EndBlock; } // FIXME: Mapping the index could fail when there is too many ivar names. If we're // compiling for a branch stub that can cause the exception to be thrown from the // wrong PC. let ivar_index = unsafe { rb_obj_ensure_iv_index_mapping(comptime_receiver, ivar_name) }.as_usize(); // Pop receiver if it's on the temp stack if recv_opnd != SelfOpnd { ctx.stack_pop(1); } if USE_RVARGC != 0 { // Check that the ivar table is big enough // Check that the slot is inside the ivar table (num_slots > index) let num_slots = Opnd::mem(32, recv, ROBJECT_OFFSET_NUMIV); asm.cmp(num_slots, Opnd::UImm(ivar_index as u64)); asm.jbe(counted_exit!(ocb, side_exit, getivar_idx_out_of_range).into()); } // Compile time self is embedded and the ivar index lands within the object let test_result = unsafe { FL_TEST_RAW(comptime_receiver, VALUE(ROBJECT_EMBED.as_usize())) != VALUE(0) }; if test_result { // See ROBJECT_IVPTR() from include/ruby/internal/core/robject.h // Guard that self is embedded // TODO: BT and JC is shorter asm.comment("guard embedded getivar"); let flags_opnd = Opnd::mem(64, recv, RUBY_OFFSET_RBASIC_FLAGS); asm.test(flags_opnd, Opnd::UImm(ROBJECT_EMBED as u64)); let side_exit = counted_exit!(ocb, side_exit, getivar_megamorphic); jit_chain_guard( JCC_JZ, jit, &starting_context, asm, ocb, max_chain_depth, side_exit, ); // Load the variable let offs = ROBJECT_OFFSET_AS_ARY + (ivar_index * SIZEOF_VALUE) as i32; let ivar_opnd = Opnd::mem(64, recv, offs); // Guard that the variable is not Qundef asm.cmp(ivar_opnd, Qundef.into()); let out_val = asm.csel_e(Qnil.into(), ivar_opnd); // Push the ivar on the stack let out_opnd = ctx.stack_push(Type::Unknown); asm.mov(out_opnd, out_val); } else { // Compile time value is *not* embedded. // Guard that value is *not* embedded // See ROBJECT_IVPTR() from include/ruby/internal/core/robject.h asm.comment("guard extended getivar"); let flags_opnd = Opnd::mem(64, recv, RUBY_OFFSET_RBASIC_FLAGS); asm.test(flags_opnd, Opnd::UImm(ROBJECT_EMBED as u64)); let megamorphic_side_exit = counted_exit!(ocb, side_exit, getivar_megamorphic); jit_chain_guard( JCC_JNZ, jit, &starting_context, asm, ocb, max_chain_depth, megamorphic_side_exit, ); if USE_RVARGC == 0 { // Check that the extended table is big enough // Check that the slot is inside the extended table (num_slots > index) let num_slots = Opnd::mem(32, recv, ROBJECT_OFFSET_NUMIV); asm.cmp(num_slots, Opnd::UImm(ivar_index as u64)); asm.jbe(Target::CodePtr(counted_exit!(ocb, side_exit, getivar_idx_out_of_range))); } // Get a pointer to the extended table let tbl_opnd = asm.load(Opnd::mem(64, recv, ROBJECT_OFFSET_AS_HEAP_IVPTR)); // Read the ivar from the extended table let ivar_opnd = Opnd::mem(64, tbl_opnd, (SIZEOF_VALUE * ivar_index) as i32); // Check that the ivar is not Qundef asm.cmp(ivar_opnd, Qundef.into()); let out_val = asm.csel_ne(ivar_opnd, Qnil.into()); // Push the ivar on the stack let out_opnd = ctx.stack_push(Type::Unknown); asm.mov(out_opnd, out_val); } // Jump to next instruction. This allows guard chains to share the same successor. jump_to_next_insn(jit, ctx, asm, ocb); EndBlock } fn gen_getinstancevariable( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Defer compilation so we can specialize on a runtime `self` if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let ivar_name = jit_get_arg(jit, 0).as_u64(); let comptime_val = jit_peek_at_self(jit); let comptime_val_klass = comptime_val.class_of(); // Generate a side exit let side_exit = get_side_exit(jit, ocb, ctx); // Guard that the receiver has the same class as the one from compile time. let self_asm_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF); jit_guard_known_klass( jit, ctx, asm, ocb, comptime_val_klass, self_asm_opnd, SelfOpnd, comptime_val, GET_IVAR_MAX_DEPTH, side_exit, ); gen_get_ivar( jit, ctx, asm, ocb, GET_IVAR_MAX_DEPTH, comptime_val, ivar_name, self_asm_opnd, SelfOpnd, side_exit, ) } fn gen_setinstancevariable( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let id = jit_get_arg(jit, 0); let ic = jit_get_arg(jit, 1).as_u64(); // type IVC // Save the PC and SP because the callee may allocate // Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm); // Get the operands from the stack let val_opnd = ctx.stack_pop(1); // Call rb_vm_setinstancevariable(iseq, obj, id, val, ic); asm.ccall( rb_vm_setinstancevariable as *const u8, vec![ Opnd::const_ptr(jit.iseq as *const u8), Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF), Opnd::UImm(id.into()), val_opnd, Opnd::const_ptr(ic as *const u8), ] ); KeepCompiling } fn gen_defined( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let op_type = jit_get_arg(jit, 0); let obj = jit_get_arg(jit, 1); let pushval = jit_get_arg(jit, 2); // Save the PC and SP because the callee may allocate // Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm); // Get the operands from the stack let v_opnd = ctx.stack_pop(1); // Call vm_defined(ec, reg_cfp, op_type, obj, v) let def_result = asm.ccall(rb_vm_defined as *const u8, vec![EC, CFP, op_type.into(), obj.into(), v_opnd]); // if (vm_defined(ec, GET_CFP(), op_type, obj, v)) { // val = pushval; // } asm.test(def_result, Opnd::UImm(255)); let out_value = asm.csel_nz(pushval.into(), Qnil.into()); // Push the return value onto the stack let out_type = if pushval.special_const_p() { Type::UnknownImm } else { Type::Unknown }; let stack_ret = ctx.stack_push(out_type); asm.mov(stack_ret, out_value); KeepCompiling } fn gen_checktype( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let type_val = jit_get_arg(jit, 0).as_u32(); // Only three types are emitted by compile.c at the moment if let RUBY_T_STRING | RUBY_T_ARRAY | RUBY_T_HASH = type_val { let val_type = ctx.get_opnd_type(StackOpnd(0)); let val = asm.load(ctx.stack_pop(1)); // Check if we know from type information match val_type.known_value_type() { Some(value_type) => { if value_type == type_val { jit_putobject(jit, ctx, asm, Qtrue); return KeepCompiling; } else { jit_putobject(jit, ctx, asm, Qfalse); return KeepCompiling; } }, _ => (), } let ret = asm.new_label("ret"); if !val_type.is_heap() { // if (SPECIAL_CONST_P(val)) { // Return Qfalse via REG1 if not on heap asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); asm.jnz(ret); asm.cmp(val, Opnd::UImm(Qnil.into())); asm.jbe(ret); } // Check type on object let object_type = asm.and( Opnd::mem(64, val, RUBY_OFFSET_RBASIC_FLAGS), Opnd::UImm(RUBY_T_MASK.into())); asm.cmp(object_type, Opnd::UImm(type_val.into())); let ret_opnd = asm.csel_e(Qtrue.into(), Qfalse.into()); asm.write_label(ret); let stack_ret = ctx.stack_push(Type::UnknownImm); asm.mov(stack_ret, ret_opnd); KeepCompiling } else { CantCompile } } fn gen_concatstrings( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let n = jit_get_arg(jit, 0); // Save the PC and SP because we are allocating jit_prepare_routine_call(jit, ctx, asm); let values_ptr = asm.lea(ctx.sp_opnd(-((SIZEOF_VALUE as isize) * n.as_isize()))); // call rb_str_concat_literals(long n, const VALUE *strings); let return_value = asm.ccall( rb_str_concat_literals as *const u8, vec![Opnd::UImm(n.into()), values_ptr] ); ctx.stack_pop(n.as_usize()); let stack_ret = ctx.stack_push(Type::CString); asm.mov(stack_ret, return_value); KeepCompiling } fn guard_two_fixnums(ctx: &mut Context, asm: &mut Assembler, side_exit: CodePtr) { // Get the stack operand types let arg1_type = ctx.get_opnd_type(StackOpnd(0)); let arg0_type = ctx.get_opnd_type(StackOpnd(1)); if arg0_type.is_heap() || arg1_type.is_heap() { asm.comment("arg is heap object"); asm.jmp(side_exit.into()); return; } if arg0_type != Type::Fixnum && arg0_type.is_specific() { asm.comment("arg0 not fixnum"); asm.jmp(side_exit.into()); return; } if arg1_type != Type::Fixnum && arg1_type.is_specific() { asm.comment("arg1 not fixnum"); asm.jmp(side_exit.into()); return; } assert!(!arg0_type.is_heap()); assert!(!arg1_type.is_heap()); assert!(arg0_type == Type::Fixnum || arg0_type.is_unknown()); assert!(arg1_type == Type::Fixnum || arg1_type.is_unknown()); // Get stack operands without popping them let arg1 = ctx.stack_opnd(0); let arg0 = ctx.stack_opnd(1); // If not fixnums, fall back if arg0_type != Type::Fixnum { asm.comment("guard arg0 fixnum"); asm.test(arg0, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); asm.jz(side_exit.into()); } if arg1_type != Type::Fixnum { asm.comment("guard arg1 fixnum"); asm.test(arg1, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); asm.jz(side_exit.into()); } // Set stack types in context ctx.upgrade_opnd_type(StackOpnd(0), Type::Fixnum); ctx.upgrade_opnd_type(StackOpnd(1), Type::Fixnum); } // Conditional move operation used by comparison operators type CmovFn = fn(cb: &mut Assembler, opnd0: Opnd, opnd1: Opnd) -> Opnd; fn gen_fixnum_cmp( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, cmov_op: CmovFn, ) -> CodegenStatus { // Defer compilation so we can specialize base on a runtime receiver if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let comptime_a = jit_peek_at_stack(jit, ctx, 1); let comptime_b = jit_peek_at_stack(jit, ctx, 0); if comptime_a.fixnum_p() && comptime_b.fixnum_p() { // Create a side-exit to fall back to the interpreter // Note: we generate the side-exit before popping operands from the stack let side_exit = get_side_exit(jit, ocb, ctx); if !assume_bop_not_redefined(jit, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_LT) { return CantCompile; } // Check that both operands are fixnums guard_two_fixnums(ctx, asm, side_exit); // Get the operands from the stack let arg1 = ctx.stack_pop(1); let arg0 = ctx.stack_pop(1); // Compare the arguments asm.cmp(arg0, arg1); let bool_opnd = cmov_op(asm, Qtrue.into(), Qfalse.into()); // Push the output on the stack let dst = ctx.stack_push(Type::Unknown); asm.mov(dst, bool_opnd); KeepCompiling } else { gen_opt_send_without_block(jit, ctx, asm, ocb) } } fn gen_opt_lt( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { gen_fixnum_cmp(jit, ctx, asm, ocb, Assembler::csel_l) } fn gen_opt_le( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { gen_fixnum_cmp(jit, ctx, asm, ocb, Assembler::csel_le) } fn gen_opt_ge( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { gen_fixnum_cmp(jit, ctx, asm, ocb, Assembler::csel_ge) } fn gen_opt_gt( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { gen_fixnum_cmp(jit, ctx, asm, ocb, Assembler::csel_g) } // Implements specialized equality for either two fixnum or two strings // Returns true if code was generated, otherwise false fn gen_equality_specialized( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, side_exit: CodePtr, ) -> bool { let comptime_a = jit_peek_at_stack(jit, ctx, 1); let comptime_b = jit_peek_at_stack(jit, ctx, 0); let a_opnd = ctx.stack_opnd(1); let b_opnd = ctx.stack_opnd(0); if comptime_a.fixnum_p() && comptime_b.fixnum_p() { if !assume_bop_not_redefined(jit, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_EQ) { // if overridden, emit the generic version return false; } guard_two_fixnums(ctx, asm, side_exit); asm.cmp(a_opnd, b_opnd); let val = asm.csel_ne(Qfalse.into(), Qtrue.into()); // Push the output on the stack ctx.stack_pop(2); let dst = ctx.stack_push(Type::UnknownImm); asm.mov(dst, val); true } else if unsafe { comptime_a.class_of() == rb_cString && comptime_b.class_of() == rb_cString } { if !assume_bop_not_redefined(jit, ocb, STRING_REDEFINED_OP_FLAG, BOP_EQ) { // if overridden, emit the generic version return false; } // Guard that a is a String jit_guard_known_klass( jit, ctx, asm, ocb, unsafe { rb_cString }, a_opnd, StackOpnd(1), comptime_a, SEND_MAX_DEPTH, side_exit, ); let equal = asm.new_label("equal"); let ret = asm.new_label("ret"); // If they are equal by identity, return true asm.cmp(a_opnd, b_opnd); asm.je(equal); // Otherwise guard that b is a T_STRING (from type info) or String (from runtime guard) let btype = ctx.get_opnd_type(StackOpnd(0)); if btype.known_value_type() != Some(RUBY_T_STRING) { // Note: any T_STRING is valid here, but we check for a ::String for simplicity // To pass a mutable static variable (rb_cString) requires an unsafe block jit_guard_known_klass( jit, ctx, asm, ocb, unsafe { rb_cString }, b_opnd, StackOpnd(0), comptime_b, SEND_MAX_DEPTH, side_exit, ); } // Call rb_str_eql_internal(a, b) let val = asm.ccall(rb_str_eql_internal as *const u8, vec![a_opnd, b_opnd]); // Push the output on the stack ctx.stack_pop(2); let dst = ctx.stack_push(Type::UnknownImm); asm.mov(dst, val); asm.jmp(ret); asm.write_label(equal); asm.mov(dst, Qtrue.into()); asm.write_label(ret); true } else { false } } fn gen_opt_eq( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Defer compilation so we can specialize base on a runtime receiver if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } // Create a side-exit to fall back to the interpreter let side_exit = get_side_exit(jit, ocb, ctx); if gen_equality_specialized(jit, ctx, asm, ocb, side_exit) { jump_to_next_insn(jit, ctx, asm, ocb); EndBlock } else { gen_opt_send_without_block(jit, ctx, asm, ocb) } } fn gen_opt_neq( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // opt_neq is passed two rb_call_data as arguments: // first for ==, second for != let cd = jit_get_arg(jit, 1).as_ptr(); return gen_send_general(jit, ctx, asm, ocb, cd, None); } fn gen_opt_aref( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let cd: *const rb_call_data = jit_get_arg(jit, 0).as_ptr(); let argc = unsafe { vm_ci_argc((*cd).ci) }; // Only JIT one arg calls like `ary[6]` if argc != 1 { gen_counter_incr!(asm, oaref_argc_not_one); return CantCompile; } // Defer compilation so we can specialize base on a runtime receiver if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } // Remember the context on entry for adding guard chains let starting_context = *ctx; // Specialize base on compile time values let comptime_idx = jit_peek_at_stack(jit, ctx, 0); let comptime_recv = jit_peek_at_stack(jit, ctx, 1); // Create a side-exit to fall back to the interpreter let side_exit = get_side_exit(jit, ocb, ctx); if comptime_recv.class_of() == unsafe { rb_cArray } && comptime_idx.fixnum_p() { if !assume_bop_not_redefined(jit, ocb, ARRAY_REDEFINED_OP_FLAG, BOP_AREF) { return CantCompile; } // Pop the stack operands let idx_opnd = ctx.stack_pop(1); let recv_opnd = ctx.stack_pop(1); let recv_reg = asm.load(recv_opnd); // if (SPECIAL_CONST_P(recv)) { // Bail if receiver is not a heap object asm.test(recv_reg, (RUBY_IMMEDIATE_MASK as u64).into()); asm.jnz(side_exit.into()); asm.cmp(recv_reg, Qfalse.into()); asm.je(side_exit.into()); asm.cmp(recv_reg, Qnil.into()); asm.je(side_exit.into()); // Bail if recv has a class other than ::Array. // BOP_AREF check above is only good for ::Array. asm.cmp(unsafe { rb_cArray }.into(), Opnd::mem(64, recv_reg, RUBY_OFFSET_RBASIC_KLASS)); jit_chain_guard( JCC_JNE, jit, &starting_context, asm, ocb, OPT_AREF_MAX_CHAIN_DEPTH, side_exit, ); // Bail if idx is not a FIXNUM let idx_reg = asm.load(idx_opnd); asm.test(idx_reg, (RUBY_FIXNUM_FLAG as u64).into()); asm.jz(counted_exit!(ocb, side_exit, oaref_arg_not_fixnum).into()); // Call VALUE rb_ary_entry_internal(VALUE ary, long offset). // It never raises or allocates, so we don't need to write to cfp->pc. { let idx_reg = asm.rshift(idx_reg, Opnd::UImm(1)); // Convert fixnum to int let val = asm.ccall(rb_ary_entry_internal as *const u8, vec![recv_opnd, idx_reg]); // Push the return value onto the stack let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, val); } // Jump to next instruction. This allows guard chains to share the same successor. jump_to_next_insn(jit, ctx, asm, ocb); return EndBlock; } else if comptime_recv.class_of() == unsafe { rb_cHash } { if !assume_bop_not_redefined(jit, ocb, HASH_REDEFINED_OP_FLAG, BOP_AREF) { return CantCompile; } let recv_opnd = ctx.stack_opnd(1); // Guard that the receiver is a hash jit_guard_known_klass( jit, ctx, asm, ocb, unsafe { rb_cHash }, recv_opnd, StackOpnd(1), comptime_recv, OPT_AREF_MAX_CHAIN_DEPTH, side_exit, ); // Prepare to call rb_hash_aref(). It might call #hash on the key. jit_prepare_routine_call(jit, ctx, asm); // Call rb_hash_aref let key_opnd = ctx.stack_opnd(0); let recv_opnd = ctx.stack_opnd(1); let val = asm.ccall(rb_hash_aref as *const u8, vec![recv_opnd, key_opnd]); // Pop the key and the receiver ctx.stack_pop(2); // Push the return value onto the stack let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, val); // Jump to next instruction. This allows guard chains to share the same successor. jump_to_next_insn(jit, ctx, asm, ocb); EndBlock } else { // General case. Call the [] method. gen_opt_send_without_block(jit, ctx, asm, ocb) } } fn gen_opt_aset( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Defer compilation so we can specialize on a runtime `self` if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let comptime_recv = jit_peek_at_stack(jit, ctx, 2); let comptime_key = jit_peek_at_stack(jit, ctx, 1); // Get the operands from the stack let recv = ctx.stack_opnd(2); let key = ctx.stack_opnd(1); let val = ctx.stack_opnd(0); if comptime_recv.class_of() == unsafe { rb_cArray } && comptime_key.fixnum_p() { let side_exit = get_side_exit(jit, ocb, ctx); // Guard receiver is an Array jit_guard_known_klass( jit, ctx, asm, ocb, unsafe { rb_cArray }, recv, StackOpnd(2), comptime_recv, SEND_MAX_DEPTH, side_exit, ); // Guard key is a fixnum jit_guard_known_klass( jit, ctx, asm, ocb, unsafe { rb_cInteger }, key, StackOpnd(1), comptime_key, SEND_MAX_DEPTH, side_exit, ); // We might allocate or raise jit_prepare_routine_call(jit, ctx, asm); // Call rb_ary_store let recv = ctx.stack_opnd(2); let key = asm.load(ctx.stack_opnd(1)); let key = asm.rshift(key, Opnd::UImm(1)); // FIX2LONG(key) let val = ctx.stack_opnd(0); asm.ccall(rb_ary_store as *const u8, vec![recv, key, val]); // rb_ary_store returns void // stored value should still be on stack let val = asm.load(ctx.stack_opnd(0)); // Push the return value onto the stack ctx.stack_pop(3); let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, val); jump_to_next_insn(jit, ctx, asm, ocb); return EndBlock; } else if comptime_recv.class_of() == unsafe { rb_cHash } { let side_exit = get_side_exit(jit, ocb, ctx); // Guard receiver is a Hash jit_guard_known_klass( jit, ctx, asm, ocb, unsafe { rb_cHash }, recv, StackOpnd(2), comptime_recv, SEND_MAX_DEPTH, side_exit, ); // We might allocate or raise jit_prepare_routine_call(jit, ctx, asm); // Call rb_hash_aset let recv = ctx.stack_opnd(2); let key = ctx.stack_opnd(1); let val = ctx.stack_opnd(0); let ret = asm.ccall(rb_hash_aset as *const u8, vec![recv, key, val]); // Push the return value onto the stack ctx.stack_pop(3); let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, ret); jump_to_next_insn(jit, ctx, asm, ocb); EndBlock } else { gen_opt_send_without_block(jit, ctx, asm, ocb) } } fn gen_opt_and( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Defer compilation so we can specialize on a runtime `self` if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let comptime_a = jit_peek_at_stack(jit, ctx, 1); let comptime_b = jit_peek_at_stack(jit, ctx, 0); if comptime_a.fixnum_p() && comptime_b.fixnum_p() { // Create a side-exit to fall back to the interpreter // Note: we generate the side-exit before popping operands from the stack let side_exit = get_side_exit(jit, ocb, ctx); if !assume_bop_not_redefined(jit, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_AND) { return CantCompile; } // Check that both operands are fixnums guard_two_fixnums(ctx, asm, side_exit); // Get the operands and destination from the stack let arg1 = ctx.stack_pop(1); let arg0 = ctx.stack_pop(1); // Do the bitwise and arg0 & arg1 let val = asm.and(arg0, arg1); // Push the output on the stack let dst = ctx.stack_push(Type::Fixnum); asm.store(dst, val); KeepCompiling } else { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } } fn gen_opt_or( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Defer compilation so we can specialize on a runtime `self` if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let comptime_a = jit_peek_at_stack(jit, ctx, 1); let comptime_b = jit_peek_at_stack(jit, ctx, 0); if comptime_a.fixnum_p() && comptime_b.fixnum_p() { // Create a side-exit to fall back to the interpreter // Note: we generate the side-exit before popping operands from the stack let side_exit = get_side_exit(jit, ocb, ctx); if !assume_bop_not_redefined(jit, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_OR) { return CantCompile; } // Check that both operands are fixnums guard_two_fixnums(ctx, asm, side_exit); // Get the operands and destination from the stack let arg1 = ctx.stack_pop(1); let arg0 = ctx.stack_pop(1); // Do the bitwise or arg0 | arg1 let val = asm.or(arg0, arg1); // Push the output on the stack let dst = ctx.stack_push(Type::Fixnum); asm.store(dst, val); KeepCompiling } else { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } } fn gen_opt_minus( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Defer compilation so we can specialize on a runtime `self` if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let comptime_a = jit_peek_at_stack(jit, ctx, 1); let comptime_b = jit_peek_at_stack(jit, ctx, 0); if comptime_a.fixnum_p() && comptime_b.fixnum_p() { // Create a side-exit to fall back to the interpreter // Note: we generate the side-exit before popping operands from the stack let side_exit = get_side_exit(jit, ocb, ctx); if !assume_bop_not_redefined(jit, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) { return CantCompile; } // Check that both operands are fixnums guard_two_fixnums(ctx, asm, side_exit); // Get the operands and destination from the stack let arg1 = ctx.stack_pop(1); let arg0 = ctx.stack_pop(1); // Subtract arg0 - arg1 and test for overflow let val_untag = asm.sub(arg0, arg1); asm.jo(side_exit.into()); let val = asm.add(val_untag, Opnd::Imm(1)); // Push the output on the stack let dst = ctx.stack_push(Type::Fixnum); asm.store(dst, val); KeepCompiling } else { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } } fn gen_opt_mult( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } fn gen_opt_div( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } fn gen_opt_mod( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Defer compilation so we can specialize on a runtime `self` if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let comptime_a = jit_peek_at_stack(jit, ctx, 1); let comptime_b = jit_peek_at_stack(jit, ctx, 0); if comptime_a.fixnum_p() && comptime_b.fixnum_p() { // Create a side-exit to fall back to the interpreter // Note: we generate the side-exit before popping operands from the stack let side_exit = get_side_exit(jit, ocb, ctx); if !assume_bop_not_redefined(jit, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_MOD) { return CantCompile; } // Check that both operands are fixnums guard_two_fixnums(ctx, asm, side_exit); // Get the operands and destination from the stack let arg1 = ctx.stack_pop(1); let arg0 = ctx.stack_pop(1); // Check for arg0 % 0 asm.cmp(arg1, Opnd::Imm(VALUE::fixnum_from_usize(0).as_i64())); asm.je(side_exit.into()); // Call rb_fix_mod_fix(VALUE recv, VALUE obj) let ret = asm.ccall(rb_fix_mod_fix as *const u8, vec![arg0, arg1]); // Push the return value onto the stack let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, ret); KeepCompiling } else { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } } fn gen_opt_ltlt( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } fn gen_opt_nil_p( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } fn gen_opt_empty_p( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } fn gen_opt_succ( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Delegate to send, call the method on the recv gen_opt_send_without_block(jit, ctx, asm, ocb) } fn gen_opt_str_freeze( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { if !assume_bop_not_redefined(jit, ocb, STRING_REDEFINED_OP_FLAG, BOP_FREEZE) { return CantCompile; } let str = jit_get_arg(jit, 0); // Push the return value onto the stack let stack_ret = ctx.stack_push(Type::CString); asm.mov(stack_ret, str.into()); KeepCompiling } fn gen_opt_str_uminus( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { if !assume_bop_not_redefined(jit, ocb, STRING_REDEFINED_OP_FLAG, BOP_UMINUS) { return CantCompile; } let str = jit_get_arg(jit, 0); // Push the return value onto the stack let stack_ret = ctx.stack_push(Type::CString); asm.mov(stack_ret, str.into()); KeepCompiling } fn gen_opt_not( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { return gen_opt_send_without_block(jit, ctx, asm, ocb); } fn gen_opt_size( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { return gen_opt_send_without_block(jit, ctx, asm, ocb); } fn gen_opt_length( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { return gen_opt_send_without_block(jit, ctx, asm, ocb); } fn gen_opt_regexpmatch2( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { return gen_opt_send_without_block(jit, ctx, asm, ocb); } fn gen_opt_case_dispatch( _jit: &mut JITState, ctx: &mut Context, _asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // Normally this instruction would lookup the key in a hash and jump to an // offset based on that. // Instead we can take the fallback case and continue with the next // instruction. // We'd hope that our jitted code will be sufficiently fast without the // hash lookup, at least for small hashes, but it's worth revisiting this // assumption in the future. ctx.stack_pop(1); KeepCompiling // continue with the next instruction } fn gen_branchif_branch( asm: &mut Assembler, target0: CodePtr, target1: Option, shape: BranchShape, ) { assert!(target1 != None); match shape { BranchShape::Next0 => { asm.jz(target1.unwrap().into()); } BranchShape::Next1 => { asm.jnz(target0.into()); } BranchShape::Default => { asm.jnz(target0.into()); asm.jmp(target1.unwrap().into()); } } } fn gen_branchif( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let jump_offset = jit_get_arg(jit, 0).as_i32(); // Check for interrupts, but only on backward branches that may create loops if jump_offset < 0 { let side_exit = get_side_exit(jit, ocb, ctx); gen_check_ints(asm, side_exit); } // Test if any bit (outside of the Qnil bit) is on // RUBY_Qfalse /* ...0000 0000 */ // RUBY_Qnil /* ...0000 1000 */ let val_opnd = ctx.stack_pop(1); asm.test(val_opnd, Opnd::Imm(!Qnil.as_i64())); // Get the branch target instruction offsets let next_idx = jit_next_insn_idx(jit); let jump_idx = (next_idx as i32) + jump_offset; let next_block = BlockId { iseq: jit.iseq, idx: next_idx, }; let jump_block = BlockId { iseq: jit.iseq, idx: jump_idx as u32, }; // Generate the branch instructions gen_branch( jit, ctx, asm, ocb, jump_block, ctx, Some(next_block), Some(ctx), gen_branchif_branch, ); EndBlock } fn gen_branchunless_branch( asm: &mut Assembler, target0: CodePtr, target1: Option, shape: BranchShape, ) { match shape { BranchShape::Next0 => asm.jnz(target1.unwrap().into()), BranchShape::Next1 => asm.jz(target0.into()), BranchShape::Default => { asm.jz(target0.into()); asm.jmp(target1.unwrap().into()); } } } fn gen_branchunless( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let jump_offset = jit_get_arg(jit, 0).as_i32(); // Check for interrupts, but only on backward branches that may create loops if jump_offset < 0 { let side_exit = get_side_exit(jit, ocb, ctx); gen_check_ints(asm, side_exit); } // Test if any bit (outside of the Qnil bit) is on // RUBY_Qfalse /* ...0000 0000 */ // RUBY_Qnil /* ...0000 1000 */ let val_opnd = ctx.stack_pop(1); let not_qnil = !Qnil.as_i64(); asm.test(val_opnd, not_qnil.into()); // Get the branch target instruction offsets let next_idx = jit_next_insn_idx(jit) as i32; let jump_idx = next_idx + jump_offset; let next_block = BlockId { iseq: jit.iseq, idx: next_idx.try_into().unwrap(), }; let jump_block = BlockId { iseq: jit.iseq, idx: jump_idx.try_into().unwrap(), }; // Generate the branch instructions gen_branch( jit, ctx, asm, ocb, jump_block, ctx, Some(next_block), Some(ctx), gen_branchunless_branch, ); EndBlock } fn gen_branchnil_branch( asm: &mut Assembler, target0: CodePtr, target1: Option, shape: BranchShape, ) { match shape { BranchShape::Next0 => asm.jne(target1.unwrap().into()), BranchShape::Next1 => asm.je(target0.into()), BranchShape::Default => { asm.je(target0.into()); asm.jmp(target1.unwrap().into()); } } } fn gen_branchnil( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let jump_offset = jit_get_arg(jit, 0).as_i32(); // Check for interrupts, but only on backward branches that may create loops if jump_offset < 0 { let side_exit = get_side_exit(jit, ocb, ctx); gen_check_ints(asm, side_exit); } // Test if the value is Qnil // RUBY_Qnil /* ...0000 1000 */ let val_opnd = ctx.stack_pop(1); asm.cmp(val_opnd, Opnd::UImm(Qnil.into())); // Get the branch target instruction offsets let next_idx = jit_next_insn_idx(jit) as i32; let jump_idx = next_idx + jump_offset; let next_block = BlockId { iseq: jit.iseq, idx: next_idx.try_into().unwrap(), }; let jump_block = BlockId { iseq: jit.iseq, idx: jump_idx.try_into().unwrap(), }; // Generate the branch instructions gen_branch( jit, ctx, asm, ocb, jump_block, ctx, Some(next_block), Some(ctx), gen_branchnil_branch, ); EndBlock } fn gen_jump( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let jump_offset = jit_get_arg(jit, 0).as_i32(); // Check for interrupts, but only on backward branches that may create loops if jump_offset < 0 { let side_exit = get_side_exit(jit, ocb, ctx); gen_check_ints(asm, side_exit); } // Get the branch target instruction offsets let jump_idx = (jit_next_insn_idx(jit) as i32) + jump_offset; let jump_block = BlockId { iseq: jit.iseq, idx: jump_idx as u32, }; // Generate the jump instruction gen_direct_jump(jit, ctx, jump_block, asm); EndBlock } /// Guard that self or a stack operand has the same class as `known_klass`, using /// `sample_instance` to speculate about the shape of the runtime value. /// FIXNUM and on-heap integers are treated as if they have distinct classes, and /// the guard generated for one will fail for the other. /// /// Recompile as contingency if possible, or take side exit a last resort. fn jit_guard_known_klass( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, known_klass: VALUE, obj_opnd: Opnd, insn_opnd: InsnOpnd, sample_instance: VALUE, max_chain_depth: i32, side_exit: CodePtr, ) { let val_type = ctx.get_opnd_type(insn_opnd); if val_type.known_class() == Some(known_klass) { // We already know from type information that this is a match return; } if unsafe { known_klass == rb_cNilClass } { assert!(!val_type.is_heap()); assert!(val_type.is_unknown()); asm.comment("guard object is nil"); asm.cmp(obj_opnd, Qnil.into()); jit_chain_guard(JCC_JNE, jit, ctx, asm, ocb, max_chain_depth, side_exit); ctx.upgrade_opnd_type(insn_opnd, Type::Nil); } else if unsafe { known_klass == rb_cTrueClass } { assert!(!val_type.is_heap()); assert!(val_type.is_unknown()); asm.comment("guard object is true"); asm.cmp(obj_opnd, Qtrue.into()); jit_chain_guard(JCC_JNE, jit, ctx, asm, ocb, max_chain_depth, side_exit); ctx.upgrade_opnd_type(insn_opnd, Type::True); } else if unsafe { known_klass == rb_cFalseClass } { assert!(!val_type.is_heap()); assert!(val_type.is_unknown()); asm.comment("guard object is false"); assert!(Qfalse.as_i32() == 0); asm.test(obj_opnd, obj_opnd); jit_chain_guard(JCC_JNZ, jit, ctx, asm, ocb, max_chain_depth, side_exit); ctx.upgrade_opnd_type(insn_opnd, Type::False); } else if unsafe { known_klass == rb_cInteger } && sample_instance.fixnum_p() { // We will guard fixnum and bignum as though they were separate classes // BIGNUM can be handled by the general else case below assert!(val_type.is_unknown()); asm.comment("guard object is fixnum"); asm.test(obj_opnd, Opnd::Imm(RUBY_FIXNUM_FLAG as i64)); jit_chain_guard(JCC_JZ, jit, ctx, asm, ocb, max_chain_depth, side_exit); ctx.upgrade_opnd_type(insn_opnd, Type::Fixnum); } else if unsafe { known_klass == rb_cSymbol } && sample_instance.static_sym_p() { assert!(!val_type.is_heap()); // We will guard STATIC vs DYNAMIC as though they were separate classes // DYNAMIC symbols can be handled by the general else case below if val_type != Type::ImmSymbol || !val_type.is_imm() { assert!(val_type.is_unknown()); asm.comment("guard object is static symbol"); assert!(RUBY_SPECIAL_SHIFT == 8); asm.cmp(obj_opnd, Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); jit_chain_guard(JCC_JNE, jit, ctx, asm, ocb, max_chain_depth, side_exit); ctx.upgrade_opnd_type(insn_opnd, Type::ImmSymbol); } } else if unsafe { known_klass == rb_cFloat } && sample_instance.flonum_p() { assert!(!val_type.is_heap()); if val_type != Type::Flonum || !val_type.is_imm() { assert!(val_type.is_unknown()); // We will guard flonum vs heap float as though they were separate classes asm.comment("guard object is flonum"); let flag_bits = asm.and(obj_opnd, Opnd::UImm(RUBY_FLONUM_MASK as u64)); asm.cmp(flag_bits, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); jit_chain_guard(JCC_JNE, jit, ctx, asm, ocb, max_chain_depth, side_exit); ctx.upgrade_opnd_type(insn_opnd, Type::Flonum); } } else if unsafe { FL_TEST(known_klass, VALUE(RUBY_FL_SINGLETON as usize)) != VALUE(0) && sample_instance == rb_attr_get(known_klass, id__attached__ as ID) } { // Singleton classes are attached to one specific object, so we can // avoid one memory access (and potentially the is_heap check) by // looking for the expected object directly. // Note that in case the sample instance has a singleton class that // doesn't attach to the sample instance, it means the sample instance // has an empty singleton class that hasn't been materialized yet. In // this case, comparing against the sample instance doesn't guarantee // that its singleton class is empty, so we can't avoid the memory // access. As an example, `Object.new.singleton_class` is an object in // this situation. asm.comment("guard known object with singleton class"); asm.cmp(obj_opnd, sample_instance.into()); jit_chain_guard(JCC_JNE, jit, ctx, asm, ocb, max_chain_depth, side_exit); } else if val_type == Type::CString && unsafe { known_klass == rb_cString } { // guard elided because the context says we've already checked unsafe { assert_eq!(sample_instance.class_of(), rb_cString, "context says class is exactly ::String") }; } else { assert!(!val_type.is_imm()); // Check that the receiver is a heap object // Note: if we get here, the class doesn't have immediate instances. if !val_type.is_heap() { asm.comment("guard not immediate"); assert!(Qfalse.as_i32() < Qnil.as_i32()); asm.test(obj_opnd, Opnd::Imm(RUBY_IMMEDIATE_MASK as i64)); jit_chain_guard(JCC_JNZ, jit, ctx, asm, ocb, max_chain_depth, side_exit); asm.cmp(obj_opnd, Qnil.into()); jit_chain_guard(JCC_JBE, jit, ctx, asm, ocb, max_chain_depth, side_exit); ctx.upgrade_opnd_type(insn_opnd, Type::UnknownHeap); } // If obj_opnd isn't already a register, load it. let obj_opnd = match obj_opnd { Opnd::Reg(_) => obj_opnd, _ => asm.load(obj_opnd), }; let klass_opnd = Opnd::mem(64, obj_opnd, RUBY_OFFSET_RBASIC_KLASS); // Bail if receiver class is different from known_klass // TODO: jit_mov_gc_ptr keeps a strong reference, which leaks the class. asm.comment("guard known class"); asm.cmp(klass_opnd, known_klass.into()); jit_chain_guard(JCC_JNE, jit, ctx, asm, ocb, max_chain_depth, side_exit); if known_klass == unsafe { rb_cString } { ctx.upgrade_opnd_type(insn_opnd, Type::CString); } } } // Generate ancestry guard for protected callee. // Calls to protected callees only go through when self.is_a?(klass_that_defines_the_callee). fn jit_protected_callee_ancestry_guard( jit: &mut JITState, asm: &mut Assembler, ocb: &mut OutlinedCb, cme: *const rb_callable_method_entry_t, side_exit: CodePtr, ) { // See vm_call_method(). let def_class = unsafe { (*cme).defined_class }; // Note: PC isn't written to current control frame as rb_is_kind_of() shouldn't raise. // VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass); let val = asm.ccall( rb_obj_is_kind_of as *mut u8, vec![ Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF), def_class.into(), ], ); asm.test(val, val); asm.jz(counted_exit!(ocb, side_exit, send_se_protected_check_failed).into()) } // Codegen for rb_obj_not(). // Note, caller is responsible for generating all the right guards, including // arity guards. fn jit_rb_obj_not( _jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option, _argc: i32, _known_recv_class: *const VALUE, ) -> bool { let recv_opnd = ctx.get_opnd_type(StackOpnd(0)); match recv_opnd.known_truthy() { Some(false) => { asm.comment("rb_obj_not(nil_or_false)"); ctx.stack_pop(1); let out_opnd = ctx.stack_push(Type::True); asm.mov(out_opnd, Qtrue.into()); }, Some(true) => { // Note: recv_opnd != Type::Nil && recv_opnd != Type::False. asm.comment("rb_obj_not(truthy)"); ctx.stack_pop(1); let out_opnd = ctx.stack_push(Type::False); asm.mov(out_opnd, Qfalse.into()); }, _ => { return false; }, } true } // Codegen for rb_true() fn jit_rb_true( _jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option, _argc: i32, _known_recv_class: *const VALUE, ) -> bool { asm.comment("nil? == true"); ctx.stack_pop(1); let stack_ret = ctx.stack_push(Type::True); asm.mov(stack_ret, Qtrue.into()); true } // Codegen for rb_false() fn jit_rb_false( _jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option, _argc: i32, _known_recv_class: *const VALUE, ) -> bool { asm.comment("nil? == false"); ctx.stack_pop(1); let stack_ret = ctx.stack_push(Type::False); asm.mov(stack_ret, Qfalse.into()); true } // Codegen for rb_obj_equal() // object identity comparison fn jit_rb_obj_equal( _jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option, _argc: i32, _known_recv_class: *const VALUE, ) -> bool { asm.comment("equal?"); let obj1 = ctx.stack_pop(1); let obj2 = ctx.stack_pop(1); asm.cmp(obj1, obj2); let ret_opnd = asm.csel_e(Qtrue.into(), Qfalse.into()); let stack_ret = ctx.stack_push(Type::UnknownImm); asm.mov(stack_ret, ret_opnd); true } /// If string is frozen, duplicate it to get a non-frozen string. Otherwise, return it. fn jit_rb_str_uplus( _jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option, _argc: i32, _known_recv_class: *const VALUE, ) -> bool { asm.comment("Unary plus on string"); let recv_opnd = asm.load(ctx.stack_pop(1)); let flags_opnd = asm.load(Opnd::mem(64, recv_opnd, RUBY_OFFSET_RBASIC_FLAGS)); asm.test(flags_opnd, Opnd::Imm(RUBY_FL_FREEZE as i64)); let ret_label = asm.new_label("stack_ret"); // If the string isn't frozen, we just return it. It's already in REG0. asm.jz(ret_label); // Str is frozen - duplicate let ret_opnd = asm.ccall(rb_str_dup as *const u8, vec![recv_opnd]); asm.write_label(ret_label); // We guard for an exact-class match on the receiver of rb_cString let stack_ret = ctx.stack_push(Type::CString); asm.mov(stack_ret, ret_opnd); true } fn jit_rb_str_bytesize( _jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option, _argc: i32, _known_recv_class: *const VALUE, ) -> bool { asm.comment("String#bytesize"); let recv = ctx.stack_pop(1); let ret_opnd = asm.ccall(rb_str_bytesize as *const u8, vec![recv]); let out_opnd = ctx.stack_push(Type::Fixnum); asm.mov(out_opnd, ret_opnd); true } /* // Codegen for rb_str_to_s() // When String#to_s is called on a String instance, the method returns self and // most of the overhead comes from setting up the method call. We observed that // this situation happens a lot in some workloads. fn jit_rb_str_to_s( _jit: &mut JITState, _ctx: &mut Context, cb: &mut CodeBlock, _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option, _argc: i32, known_recv_class: *const VALUE, ) -> bool { if !known_recv_class.is_null() && unsafe { *known_recv_class == rb_cString } { add_comment(cb, "to_s on plain string"); // The method returns the receiver, which is already on the stack. // No stack movement. return true; } false } // Codegen for rb_str_concat() -- *not* String#concat // Frequently strings are concatenated using "out_str << next_str". // This is common in Erb and similar templating languages. fn jit_rb_str_concat( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option, _argc: i32, _known_recv_class: *const VALUE, ) -> bool { // The << operator can accept integer codepoints for characters // as the argument. We only specially optimise string arguments. // If the peeked-at compile time argument is something other than // a string, assume it won't be a string later either. let comptime_arg = jit_peek_at_stack(jit, ctx, 0); if ! unsafe { RB_TYPE_P(comptime_arg, RUBY_T_STRING) } { return false; } // Generate a side exit let side_exit = get_side_exit(jit, ocb, ctx); // Guard that the argument is of class String at runtime. let insn_opnd = StackOpnd(0); let arg_opnd = asm.load(ctx.stack_opnd(0)); let arg_type = ctx.get_opnd_type(insn_opnd); if arg_type != Type::CString && arg_type != Type::TString { if !arg_type.is_heap() { asm.comment("guard arg not immediate"); asm.test(REG0, imm_opnd(RUBY_IMMEDIATE_MASK as i64)); asm.jnz(Target::CodePtr(side_exit)); asm.cmp(arg_opnd, Qnil.into()); asm.jbe(Target::CodePtr(side_exit)); ctx.upgrade_opnd_type(insn_opnd, Type::UnknownHeap); } guard_object_is_string(cb, REG0, REG1, side_exit); // We know this has type T_STRING, but not necessarily that it's a ::String ctx.upgrade_opnd_type(insn_opnd, Type::TString); } let concat_arg = ctx.stack_pop(1); let recv = ctx.stack_pop(1); // Test if string encodings differ. If different, use rb_str_append. If the same, // use rb_yjit_str_simple_append, which calls rb_str_cat. asm.comment("<< on strings"); // Both rb_str_append and rb_yjit_str_simple_append take identical args let ccall_args = vec![recv, concat_arg]; // Take receiver's object flags XOR arg's flags. If any // string-encoding flags are different between the two, // the encodings don't match. let flags_xor = asm.xor( Opnd::mem(64, asm.load(recv), RUBY_OFFSET_RBASIC_FLAGS), Opnd::mem(64, asm.load(concat_arg), RUBY_OFFSET_RBASIC_FLAGS) ); asm.test(flags_xor, Opnd::UImm(RUBY_ENCODING_MASK as u64)); let enc_mismatch = asm.new_label("enc_mismatch"); asm.jnz(enc_mismatch); // If encodings match, call the simple append function and jump to return let ret_opnd = asm.ccall(rb_yjit_str_simple_append as *const u8, ccall_args); let ret_label = asm.new_label("stack_return"); asm.jmp(ret_label); // If encodings are different, use a slower encoding-aware concatenate asm.write_label(enc_mismatch); asm.ccall(rb_str_buf_append as *const u8, ccall_args); // Drop through to return asm.write_label(ret_label); let stack_ret = ctx.stack_push(Type::CString); asm.mov(stack_ret, ret_opnd); true } */ fn jit_thread_s_current( _jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, _ci: *const rb_callinfo, _cme: *const rb_callable_method_entry_t, _block: Option, _argc: i32, _known_recv_class: *const VALUE, ) -> bool { asm.comment("Thread.current"); ctx.stack_pop(1); // ec->thread_ptr let ec_thread_opnd = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_THREAD_PTR)); // thread->self let thread_self = Opnd::mem(64, ec_thread_opnd, RUBY_OFFSET_THREAD_SELF); let stack_ret = ctx.stack_push(Type::UnknownHeap); asm.mov(stack_ret, thread_self); true } // Check if we know how to codegen for a particular cfunc method fn lookup_cfunc_codegen(def: *const rb_method_definition_t) -> Option { let method_serial = unsafe { get_def_method_serial(def) }; CodegenGlobals::look_up_codegen_method(method_serial) } // Is anyone listening for :c_call and :c_return event currently? fn c_method_tracing_currently_enabled(jit: &JITState) -> bool { // Defer to C implementation in yjit.c unsafe { rb_c_method_tracing_currently_enabled(jit.ec.unwrap() as *mut rb_execution_context_struct) } } // Similar to args_kw_argv_to_hash. It is called at runtime from within the // generated assembly to build a Ruby hash of the passed keyword arguments. The // keys are the Symbol objects associated with the keywords and the values are // the actual values. In the representation, both keys and values are VALUEs. unsafe extern "C" fn build_kwhash(ci: *const rb_callinfo, sp: *const VALUE) -> VALUE { let kw_arg = vm_ci_kwarg(ci); let kw_len: usize = get_cikw_keyword_len(kw_arg).try_into().unwrap(); let hash = rb_hash_new_with_size(kw_len as u64); for kwarg_idx in 0..kw_len { let key = get_cikw_keywords_idx(kw_arg, kwarg_idx.try_into().unwrap()); let val = sp.sub(kw_len).add(kwarg_idx).read(); rb_hash_aset(hash, key, val); } hash } fn gen_send_cfunc( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, block: Option, argc: i32, recv_known_klass: *const VALUE, ) -> CodegenStatus { let cfunc = unsafe { get_cme_def_body_cfunc(cme) }; let cfunc_argc = unsafe { get_mct_argc(cfunc) }; // If the function expects a Ruby array of arguments if cfunc_argc < 0 && cfunc_argc != -1 { gen_counter_incr!(asm, send_cfunc_ruby_array_varg); return CantCompile; } let kw_arg = unsafe { vm_ci_kwarg(ci) }; let kw_arg_num = if kw_arg.is_null() { 0 } else { unsafe { get_cikw_keyword_len(kw_arg) } }; // Number of args which will be passed through to the callee // This is adjusted by the kwargs being combined into a hash. let passed_argc = if kw_arg.is_null() { argc } else { argc - kw_arg_num + 1 }; // If the argument count doesn't match if cfunc_argc >= 0 && cfunc_argc != passed_argc { gen_counter_incr!(asm, send_cfunc_argc_mismatch); return CantCompile; } // Don't JIT functions that need C stack arguments for now if cfunc_argc >= 0 && passed_argc + 1 > (C_ARG_OPNDS.len() as i32) { gen_counter_incr!(asm, send_cfunc_toomany_args); return CantCompile; } if c_method_tracing_currently_enabled(jit) { // Don't JIT if tracing c_call or c_return gen_counter_incr!(asm, send_cfunc_tracing); return CantCompile; } // Delegate to codegen for C methods if we have it. if kw_arg.is_null() { let codegen_p = lookup_cfunc_codegen(unsafe { (*cme).def }); if let Some(known_cfunc_codegen) = codegen_p { return CantCompile; /* let start_pos = cb.get_write_ptr().raw_ptr() as usize; if known_cfunc_codegen(jit, ctx, cb, ocb, ci, cme, block, argc, recv_known_klass) { let written_bytes = cb.get_write_ptr().raw_ptr() as usize - start_pos; if written_bytes < JUMP_SIZE_IN_BYTES { add_comment(cb, "Writing NOPs to leave room for later invalidation code"); nop(cb, (JUMP_SIZE_IN_BYTES - written_bytes) as u32); } // cfunc codegen generated code. Terminate the block so // there isn't multiple calls in the same block. jump_to_next_insn(jit, ctx, cb, ocb); return EndBlock; } */ } } // Create a side-exit to fall back to the interpreter let side_exit = get_side_exit(jit, ocb, ctx); // Check for interrupts gen_check_ints(asm, side_exit); // Stack overflow check // #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin) // REG_CFP <= REG_SP + 4 * SIZEOF_VALUE + sizeof(rb_control_frame_t) asm.comment("stack overflow check"); let stack_limit = asm.lea(ctx.sp_opnd((SIZEOF_VALUE * 4 + 2 * RUBY_SIZEOF_CONTROL_FRAME) as isize)); asm.cmp(CFP, stack_limit); asm.jbe(counted_exit!(ocb, side_exit, send_se_cf_overflow).into()); // Points to the receiver operand on the stack let recv = ctx.stack_opnd(argc); // Store incremented PC into current control frame in case callee raises. jit_save_pc(jit, asm); if let Some(block_iseq) = block { // Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). // VM_CFP_TO_CAPTURED_BLOCK does &cfp->self, rb_captured_block->code.iseq aliases // with cfp->block_code. asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), Opnd::UImm(block_iseq as u64)); } // Increment the stack pointer by 3 (in the callee) // sp += 3 let sp = asm.lea(ctx.sp_opnd((SIZEOF_VALUE as isize) * 3)); // Write method entry at sp[-3] // sp[-3] = me; // Put compile time cme into REG1. It's assumed to be valid because we are notified when // any cme we depend on become outdated. See yjit_method_lookup_change(). asm.mov(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -3), Opnd::UImm(cme as u64)); // Write block handler at sp[-2] // sp[-2] = block_handler; if let Some(_block_iseq) = block { // reg1 = VM_BH_FROM_ISEQ_BLOCK(VM_CFP_TO_CAPTURED_BLOCK(reg_cfp)); let cfp_self = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)); let block_handler = asm.or(cfp_self, Opnd::Imm(1)); asm.store(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -2), block_handler); } else { let dst_opnd = Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -2); asm.store(dst_opnd, Opnd::UImm(VM_BLOCK_HANDLER_NONE.into())); } // Write env flags at sp[-1] // sp[-1] = frame_type; let mut frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL; if !kw_arg.is_null() { frame_type |= VM_FRAME_FLAG_CFRAME_KW } asm.store(Opnd::mem(64, sp, SIZEOF_VALUE_I32 * -1), Opnd::UImm(frame_type.into())); // Allocate a new CFP (ec->cfp--) let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP); let new_cfp = asm.sub(ec_cfp_opnd, Opnd::UImm(RUBY_SIZEOF_CONTROL_FRAME as u64)); asm.mov(ec_cfp_opnd, new_cfp); // Setup the new frame // *cfp = (const struct rb_control_frame_struct) { // .pc = 0, // .sp = sp, // .iseq = 0, // .self = recv, // .ep = sp - 1, // .block_code = 0, // .__bp__ = sp, // }; // Can we re-use ec_cfp_opnd from above? let ec_cfp_opnd = asm.load(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP)); asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_PC), Opnd::Imm(0)); asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_SP), sp); asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_ISEQ), Opnd::Imm(0)); asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_BLOCK_CODE), Opnd::Imm(0)); asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_BP), sp); let ep = asm.sub(sp, Opnd::UImm(SIZEOF_VALUE as u64)); asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_EP), ep); asm.mov(Opnd::mem(64, ec_cfp_opnd, RUBY_OFFSET_CFP_SELF), recv); /* // Verify that we are calling the right function if (YJIT_CHECK_MODE > 0) { // TODO: will we have a YJIT_CHECK_MODE? // Call check_cfunc_dispatch mov(cb, C_ARG_REGS[0], recv); jit_mov_gc_ptr(jit, cb, C_ARG_REGS[1], (VALUE)ci); mov(cb, C_ARG_REGS[2], const_ptr_opnd((void *)cfunc->func)); jit_mov_gc_ptr(jit, cb, C_ARG_REGS[3], (VALUE)cme); call_ptr(cb, REG0, (void *)&check_cfunc_dispatch); } */ if !kw_arg.is_null() { // Build a hash from all kwargs passed asm.comment("build_kwhash"); let sp = asm.lea(ctx.sp_opnd(0)); let kwargs = asm.ccall(build_kwhash as *const u8, vec![Opnd::UImm(ci as u64), sp]); // Replace the stack location at the start of kwargs with the new hash let stack_opnd = ctx.stack_opnd(argc - passed_argc); asm.mov(stack_opnd, kwargs); } // Copy SP because REG_SP will get overwritten let sp = asm.lea(ctx.sp_opnd(0)); // Pop the C function arguments from the stack (in the caller) ctx.stack_pop((argc + 1).try_into().unwrap()); // Write interpreter SP into CFP. // Needed in case the callee yields to the block. gen_save_sp(jit, asm, ctx); // Non-variadic method let args = if cfunc_argc >= 0 { // Copy the arguments from the stack to the C argument registers // self is the 0th argument and is at index argc from the stack top (0..=passed_argc).map(|i| Opnd::mem(64, sp, -(argc + 1 - (i as i32)) * SIZEOF_VALUE_I32) ).collect() } // Variadic method else if cfunc_argc == -1 { // The method gets a pointer to the first argument // rb_f_puts(int argc, VALUE *argv, VALUE recv) vec![ Opnd::Imm(passed_argc.into()), asm.lea(Opnd::mem(64, sp, -(argc) * SIZEOF_VALUE_I32)), Opnd::mem(64, sp, -(argc + 1) * SIZEOF_VALUE_I32), ] } else { panic!("unexpected cfunc_args: {}", cfunc_argc) }; // Call the C function // VALUE ret = (cfunc->func)(recv, argv[0], argv[1]); // cfunc comes from compile-time cme->def, which we assume to be stable. // Invalidation logic is in yjit_method_lookup_change() asm.comment("call C function"); let ret = asm.ccall(unsafe { get_mct_func(cfunc) }.cast(), args); // Record code position for TracePoint patching. See full_cfunc_return(). record_global_inval_patch(asm, CodegenGlobals::get_outline_full_cfunc_return_pos()); // Push the return value on the Ruby stack let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, ret); // Pop the stack frame (ec->cfp++) // Can we reuse ec_cfp_opnd from above? let ec_cfp_opnd = Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP); let new_cfp = asm.add(ec_cfp_opnd, Opnd::UImm(RUBY_SIZEOF_CONTROL_FRAME as u64)); asm.store(ec_cfp_opnd, new_cfp); // cfunc calls may corrupt types ctx.clear_local_types(); // Note: the return block of gen_send_iseq() has ctx->sp_offset == 1 // which allows for sharing the same successor. // Jump (fall through) to the call continuation block // We do this to end the current block after the call jump_to_next_insn(jit, ctx, asm, ocb); EndBlock } fn gen_return_branch( asm: &mut Assembler, target0: CodePtr, _target1: Option, shape: BranchShape, ) { match shape { BranchShape::Next0 | BranchShape::Next1 => unreachable!(), BranchShape::Default => { asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(target0.raw_ptr())); } } } fn gen_send_iseq( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, block: Option, argc: i32, ) -> CodegenStatus { let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; let mut argc = argc; // When you have keyword arguments, there is an extra object that gets // placed on the stack the represents a bitmap of the keywords that were not // specified at the call site. We need to keep track of the fact that this // value is present on the stack in order to properly set up the callee's // stack pointer. let doing_kw_call = unsafe { get_iseq_flags_has_kw(iseq) }; let supplying_kws = unsafe { vm_ci_flag(ci) & VM_CALL_KWARG } != 0; if unsafe { vm_ci_flag(ci) } & VM_CALL_TAILCALL != 0 { // We can't handle tailcalls gen_counter_incr!(asm, send_iseq_tailcall); return CantCompile; } // No support for callees with these parameters yet as they require allocation // or complex handling. if unsafe { get_iseq_flags_has_rest(iseq) || get_iseq_flags_has_post(iseq) || get_iseq_flags_has_kwrest(iseq) } { gen_counter_incr!(asm, send_iseq_complex_callee); return CantCompile; } // If we have keyword arguments being passed to a callee that only takes // positionals, then we need to allocate a hash. For now we're going to // call that too complex and bail. if supplying_kws && !unsafe { get_iseq_flags_has_kw(iseq) } { gen_counter_incr!(asm, send_iseq_complex_callee); return CantCompile; } // If we have a method accepting no kwargs (**nil), exit if we have passed // it any kwargs. if supplying_kws && unsafe { get_iseq_flags_has_accepts_no_kwarg(iseq) } { gen_counter_incr!(asm, send_iseq_complex_callee); return CantCompile; } // For computing number of locals to set up for the callee let mut num_params = unsafe { get_iseq_body_param_size(iseq) }; // Block parameter handling. This mirrors setup_parameters_complex(). if unsafe { get_iseq_flags_has_block(iseq) } { if unsafe { get_iseq_body_local_iseq(iseq) == iseq } { num_params -= 1; } else { // In this case (param.flags.has_block && local_iseq != iseq), // the block argument is setup as a local variable and requires // materialization (allocation). Bail. gen_counter_incr!(asm, send_iseq_complex_callee); return CantCompile; } } let mut start_pc_offset = 0; let required_num = unsafe { get_iseq_body_param_lead_num(iseq) }; // This struct represents the metadata about the caller-specified // keyword arguments. let kw_arg = unsafe { vm_ci_kwarg(ci) }; let kw_arg_num = if kw_arg.is_null() { 0 } else { unsafe { get_cikw_keyword_len(kw_arg) } }; // Arity handling and optional parameter setup let opts_filled = argc - required_num - kw_arg_num; let opt_num = unsafe { get_iseq_body_param_opt_num(iseq) }; let opts_missing: i32 = opt_num - opts_filled; if opts_filled < 0 || opts_filled > opt_num { gen_counter_incr!(asm, send_iseq_arity_error); return CantCompile; } // If we have unfilled optional arguments and keyword arguments then we // would need to move adjust the arguments location to account for that. // For now we aren't handling this case. if doing_kw_call && opts_missing > 0 { gen_counter_incr!(asm, send_iseq_complex_callee); return CantCompile; } if opt_num > 0 { num_params -= opts_missing as u32; unsafe { let opt_table = get_iseq_body_param_opt_table(iseq); start_pc_offset = (*opt_table.offset(opts_filled as isize)).as_u32(); } } if doing_kw_call { // Here we're calling a method with keyword arguments and specifying // keyword arguments at this call site. // This struct represents the metadata about the callee-specified // keyword parameters. let keyword = unsafe { get_iseq_body_param_keyword(iseq) }; let keyword_num: usize = unsafe { (*keyword).num }.try_into().unwrap(); let keyword_required_num: usize = unsafe { (*keyword).required_num }.try_into().unwrap(); let mut required_kwargs_filled = 0; if keyword_num > 30 { // We have so many keywords that (1 << num) encoded as a FIXNUM // (which shifts it left one more) no longer fits inside a 32-bit // immediate. gen_counter_incr!(asm, send_iseq_complex_callee); return CantCompile; } // Check that the kwargs being passed are valid if supplying_kws { // This is the list of keyword arguments that the callee specified // in its initial declaration. // SAFETY: see compile.c for sizing of this slice. let callee_kwargs = unsafe { slice::from_raw_parts((*keyword).table, keyword_num) }; // Here we're going to build up a list of the IDs that correspond to // the caller-specified keyword arguments. If they're not in the // same order as the order specified in the callee declaration, then // we're going to need to generate some code to swap values around // on the stack. let kw_arg_keyword_len: usize = unsafe { get_cikw_keyword_len(kw_arg) }.try_into().unwrap(); let mut caller_kwargs: Vec = vec![0; kw_arg_keyword_len]; for kwarg_idx in 0..kw_arg_keyword_len { let sym = unsafe { get_cikw_keywords_idx(kw_arg, kwarg_idx.try_into().unwrap()) }; caller_kwargs[kwarg_idx] = unsafe { rb_sym2id(sym) }; } // First, we're going to be sure that the names of every // caller-specified keyword argument correspond to a name in the // list of callee-specified keyword parameters. for caller_kwarg in caller_kwargs { let search_result = callee_kwargs .iter() .enumerate() // inject element index .find(|(_, &kwarg)| kwarg == caller_kwarg); match search_result { None => { // If the keyword was never found, then we know we have a // mismatch in the names of the keyword arguments, so we need to // bail. gen_counter_incr!(asm, send_iseq_kwargs_mismatch); return CantCompile; } Some((callee_idx, _)) if callee_idx < keyword_required_num => { // Keep a count to ensure all required kwargs are specified required_kwargs_filled += 1; } _ => (), } } } assert!(required_kwargs_filled <= keyword_required_num); if required_kwargs_filled != keyword_required_num { gen_counter_incr!(asm, send_iseq_kwargs_mismatch); return CantCompile; } } // Number of locals that are not parameters let num_locals = unsafe { get_iseq_body_local_table_size(iseq) as i32 } - (num_params as i32); // Create a side-exit to fall back to the interpreter let side_exit = get_side_exit(jit, ocb, ctx); // Check for interrupts gen_check_ints(asm, side_exit); let leaf_builtin_raw = unsafe { rb_leaf_builtin_function(iseq) }; let leaf_builtin: Option<*const rb_builtin_function> = if leaf_builtin_raw.is_null() { None } else { Some(leaf_builtin_raw) }; if let (None, Some(builtin_info)) = (block, leaf_builtin) { let builtin_argc = unsafe { (*builtin_info).argc }; if builtin_argc + 1 /* for self */ + 1 /* for ec */ <= (C_ARG_OPNDS.len() as i32) { asm.comment("inlined leaf builtin"); // Call the builtin func (ec, recv, arg1, arg2, ...) let mut args = vec![EC]; // Copy self and arguments for i in 0..=builtin_argc { let stack_opnd = ctx.stack_opnd(builtin_argc - i); args.push(stack_opnd); } ctx.stack_pop((builtin_argc + 1).try_into().unwrap()); let val = asm.ccall(unsafe { (*builtin_info).func_ptr as *const u8 }, args); // Push the return value let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, val); // Note: assuming that the leaf builtin doesn't change local variables here. // Seems like a safe assumption. return KeepCompiling; } } // Stack overflow check // Note that vm_push_frame checks it against a decremented cfp, hence the multiply by 2. // #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin) asm.comment("stack overflow check"); let stack_max: i32 = unsafe { get_iseq_body_stack_max(iseq) }.try_into().unwrap(); let locals_offs = (SIZEOF_VALUE as i32) * (num_locals + stack_max) + 2 * (RUBY_SIZEOF_CONTROL_FRAME as i32); let stack_limit = asm.lea(ctx.sp_opnd(locals_offs as isize)); asm.cmp(CFP, stack_limit); asm.jbe(counted_exit!(ocb, side_exit, send_se_cf_overflow).into()); if doing_kw_call { // Here we're calling a method with keyword arguments and specifying // keyword arguments at this call site. // Number of positional arguments the callee expects before the first // keyword argument let args_before_kw = required_num + opt_num; // This struct represents the metadata about the caller-specified // keyword arguments. let ci_kwarg = unsafe { vm_ci_kwarg(ci) }; let caller_keyword_len: usize = if ci_kwarg.is_null() { 0 } else { unsafe { get_cikw_keyword_len(ci_kwarg) } .try_into() .unwrap() }; // This struct represents the metadata about the callee-specified // keyword parameters. let keyword = unsafe { get_iseq_body_param_keyword(iseq) }; asm.comment("keyword args"); // This is the list of keyword arguments that the callee specified // in its initial declaration. let callee_kwargs = unsafe { (*keyword).table }; let total_kwargs: usize = unsafe { (*keyword).num }.try_into().unwrap(); // Here we're going to build up a list of the IDs that correspond to // the caller-specified keyword arguments. If they're not in the // same order as the order specified in the callee declaration, then // we're going to need to generate some code to swap values around // on the stack. let mut caller_kwargs: Vec = vec![0; total_kwargs]; for kwarg_idx in 0..caller_keyword_len { let sym = unsafe { get_cikw_keywords_idx(ci_kwarg, kwarg_idx.try_into().unwrap()) }; caller_kwargs[kwarg_idx] = unsafe { rb_sym2id(sym) }; } let mut kwarg_idx = caller_keyword_len; let mut unspecified_bits = 0; let keyword_required_num: usize = unsafe { (*keyword).required_num }.try_into().unwrap(); for callee_idx in keyword_required_num..total_kwargs { let mut already_passed = false; let callee_kwarg = unsafe { *(callee_kwargs.offset(callee_idx.try_into().unwrap())) }; for caller_idx in 0..caller_keyword_len { if caller_kwargs[caller_idx] == callee_kwarg { already_passed = true; break; } } if !already_passed { // Reserve space on the stack for each default value we'll be // filling in (which is done in the next loop). Also increments // argc so that the callee's SP is recorded correctly. argc += 1; let default_arg = ctx.stack_push(Type::Unknown); // callee_idx - keyword->required_num is used in a couple of places below. let req_num: isize = unsafe { (*keyword).required_num }.try_into().unwrap(); let callee_idx_isize: isize = callee_idx.try_into().unwrap(); let extra_args = callee_idx_isize - req_num; //VALUE default_value = keyword->default_values[callee_idx - keyword->required_num]; let mut default_value = unsafe { *((*keyword).default_values.offset(extra_args)) }; if default_value == Qundef { // Qundef means that this value is not constant and must be // recalculated at runtime, so we record it in unspecified_bits // (Qnil is then used as a placeholder instead of Qundef). unspecified_bits |= 0x01 << extra_args; default_value = Qnil; } asm.mov(default_arg, default_value.into()); caller_kwargs[kwarg_idx] = callee_kwarg; kwarg_idx += 1; } } assert!(kwarg_idx == total_kwargs); // Next, we're going to loop through every keyword that was // specified by the caller and make sure that it's in the correct // place. If it's not we're going to swap it around with another one. for kwarg_idx in 0..total_kwargs { let kwarg_idx_isize: isize = kwarg_idx.try_into().unwrap(); let callee_kwarg = unsafe { *(callee_kwargs.offset(kwarg_idx_isize)) }; // If the argument is already in the right order, then we don't // need to generate any code since the expected value is already // in the right place on the stack. if callee_kwarg == caller_kwargs[kwarg_idx] { continue; } // In this case the argument is not in the right place, so we // need to find its position where it _should_ be and swap with // that location. for swap_idx in (kwarg_idx + 1)..total_kwargs { if callee_kwarg == caller_kwargs[swap_idx] { // First we're going to generate the code that is going // to perform the actual swapping at runtime. let swap_idx_i32: i32 = swap_idx.try_into().unwrap(); let kwarg_idx_i32: i32 = kwarg_idx.try_into().unwrap(); let offset0: u16 = (argc - 1 - swap_idx_i32 - args_before_kw) .try_into() .unwrap(); let offset1: u16 = (argc - 1 - kwarg_idx_i32 - args_before_kw) .try_into() .unwrap(); stack_swap(jit, ctx, asm, offset0, offset1); // Next we're going to do some bookkeeping on our end so // that we know the order that the arguments are // actually in now. caller_kwargs.swap(kwarg_idx, swap_idx); break; } } } // Keyword arguments cause a special extra local variable to be // pushed onto the stack that represents the parameters that weren't // explicitly given a value and have a non-constant default. let unspec_opnd = VALUE::fixnum_from_usize(unspecified_bits).as_u64(); asm.mov(ctx.stack_opnd(-1), unspec_opnd.into()); } // Points to the receiver operand on the stack let recv = ctx.stack_opnd(argc); // Store the updated SP on the current frame (pop arguments and receiver) asm.comment("store caller sp"); let caller_sp = asm.lea(ctx.sp_opnd((SIZEOF_VALUE as isize) * -((argc as isize) + 1))); asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), caller_sp); // Store the next PC in the current frame jit_save_pc(jit, asm); if let Some(block_val) = block { // Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). // VM_CFP_TO_CAPTURED_BLCOK does &cfp->self, rb_captured_block->code.iseq aliases // with cfp->block_code. asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), VALUE(block_val as usize).into()); } // Adjust the callee's stack pointer let offs = (SIZEOF_VALUE as isize) * (3 + (num_locals as isize) + if doing_kw_call { 1 } else { 0 }); let callee_sp = asm.lea(ctx.sp_opnd(offs)); // Initialize local variables to Qnil for i in 0..num_locals { let offs = (SIZEOF_VALUE as i32) * (i - num_locals - 3); asm.store(Opnd::mem(64, callee_sp, offs), Qnil.into()); } // Write the callee CME on the stack. It's assumed to be valid because we are notified when // any cme we depend on become outdated. See yjit_method_lookup_change(). // Write method entry at sp[-3] // sp[-3] = me; asm.comment("push cme, block handler, frame type"); asm.store(Opnd::mem(64, callee_sp, SIZEOF_VALUE_I32 * -3), VALUE(cme as usize).into()); // Write block handler at sp[-2] // sp[-2] = block_handler; match block { Some(_) => { // reg1 = VM_BH_FROM_ISEQ_BLOCK(VM_CFP_TO_CAPTURED_BLOCK(reg_cfp)); let block_handler = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)); let block_handler = asm.or(block_handler, 1.into()); asm.store(Opnd::mem(64, callee_sp, SIZEOF_VALUE_I32 * -2), block_handler); } None => { asm.store(Opnd::mem(64, callee_sp, SIZEOF_VALUE_I32 * -2), VM_BLOCK_HANDLER_NONE.into()); } } // Write env flags at sp[-1] // sp[-1] = frame_type; let frame_type = VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL; asm.store(Opnd::mem(64, callee_sp, SIZEOF_VALUE_I32 * -1), frame_type.into()); asm.comment("push callee control frame"); // Allocate a new CFP (ec->cfp--) let new_cfp = asm.sub(CFP, (RUBY_SIZEOF_CONTROL_FRAME as u64).into()); asm.mov(CFP, new_cfp); asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP); // Setup the new frame // *cfp = (const struct rb_control_frame_struct) { // .pc = pc, // .sp = sp, // .iseq = iseq, // .self = recv, // .ep = sp - 1, // .block_code = 0, // .__bp__ = sp, // }; asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF), recv); asm.mov(SP, callee_sp); asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), callee_sp); asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BP), callee_sp); let callee_ep = asm.sub(callee_sp, (SIZEOF_VALUE as u64).into()); asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP), callee_ep); asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), VALUE(iseq as usize).into()); asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); // No need to set cfp->pc since the callee sets it whenever calling into routines // that could look at it through jit_save_pc(). // mov(cb, REG0, const_ptr_opnd(start_pc)); // mov(cb, member_opnd(REG_CFP, rb_control_frame_t, pc), REG0); // Stub so we can return to JITted code let return_block = BlockId { iseq: jit.iseq, idx: jit_next_insn_idx(jit), }; // Create a context for the callee let mut callee_ctx = Context::new(); // Was DEFAULT_CTX // Set the argument types in the callee's context for arg_idx in 0..argc { let stack_offs: u16 = (argc - arg_idx - 1).try_into().unwrap(); let arg_type = ctx.get_opnd_type(StackOpnd(stack_offs)); callee_ctx.set_local_type(arg_idx.try_into().unwrap(), arg_type); } let recv_type = ctx.get_opnd_type(StackOpnd(argc.try_into().unwrap())); callee_ctx.upgrade_opnd_type(SelfOpnd, recv_type); // The callee might change locals through Kernel#binding and other means. ctx.clear_local_types(); // Pop arguments and receiver in return context, push the return value // After the return, sp_offset will be 1. The codegen for leave writes // the return value in case of JIT-to-JIT return. let mut return_ctx = *ctx; return_ctx.stack_pop((argc + 1).try_into().unwrap()); return_ctx.stack_push(Type::Unknown); return_ctx.set_sp_offset(1); return_ctx.reset_chain_depth(); // Write the JIT return address on the callee frame gen_branch( jit, ctx, asm, ocb, return_block, &return_ctx, Some(return_block), Some(&return_ctx), gen_return_branch, ); //print_str(cb, "calling Ruby func:"); //print_str(cb, rb_id2name(vm_ci_mid(ci))); // Directly jump to the entry point of the callee gen_direct_jump( jit, &callee_ctx, BlockId { iseq: iseq, idx: start_pc_offset, }, asm, ); EndBlock } fn gen_struct_aref( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, comptime_recv: VALUE, _comptime_recv_klass: VALUE, ) -> CodegenStatus { if unsafe { vm_ci_argc(ci) } != 0 { return CantCompile; } let off: i32 = unsafe { get_cme_def_body_optimized_index(cme) } .try_into() .unwrap(); // Confidence checks assert!(unsafe { RB_TYPE_P(comptime_recv, RUBY_T_STRUCT) }); assert!((off as i64) < unsafe { RSTRUCT_LEN(comptime_recv) }); // We are going to use an encoding that takes a 4-byte immediate which // limits the offset to INT32_MAX. { let native_off = (off as i64) * (SIZEOF_VALUE as i64); if native_off > (i32::MAX as i64) { return CantCompile; } } // All structs from the same Struct class should have the same // length. So if our comptime_recv is embedded all runtime // structs of the same class should be as well, and the same is // true of the converse. let embedded = unsafe { FL_TEST_RAW(comptime_recv, VALUE(RSTRUCT_EMBED_LEN_MASK)) }; asm.comment("struct aref"); let recv = asm.load(ctx.stack_pop(1)); let val = if embedded != VALUE(0) { Opnd::mem(64, recv, RUBY_OFFSET_RSTRUCT_AS_ARY + ((SIZEOF_VALUE as i32) * off)) } else { let rstruct_ptr = asm.load(Opnd::mem(64, recv, RUBY_OFFSET_RSTRUCT_AS_HEAP_PTR)); Opnd::mem(64, rstruct_ptr, (SIZEOF_VALUE as i32) * off) }; let ret = ctx.stack_push(Type::Unknown); asm.mov(ret, val); jump_to_next_insn(jit, ctx, asm, ocb); EndBlock } fn gen_struct_aset( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, comptime_recv: VALUE, _comptime_recv_klass: VALUE, ) -> CodegenStatus { if unsafe { vm_ci_argc(ci) } != 1 { return CantCompile; } let off: i32 = unsafe { get_cme_def_body_optimized_index(cme) } .try_into() .unwrap(); // Confidence checks assert!(unsafe { RB_TYPE_P(comptime_recv, RUBY_T_STRUCT) }); assert!((off as i64) < unsafe { RSTRUCT_LEN(comptime_recv) }); asm.comment("struct aset"); let val = ctx.stack_pop(1); let recv = ctx.stack_pop(1); let val = asm.ccall(RSTRUCT_SET as *const u8, vec![recv, (off as i64).into(), val]); let ret = ctx.stack_push(Type::Unknown); asm.mov(ret, val); jump_to_next_insn(jit, ctx, asm, ocb); EndBlock } fn gen_send_general( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, cd: *const rb_call_data, block: Option, ) -> CodegenStatus { // Relevant definitions: // rb_execution_context_t : vm_core.h // invoker, cfunc logic : method.h, vm_method.c // rb_callinfo : vm_callinfo.h // rb_callable_method_entry_t : method.h // vm_call_cfunc_with_frame : vm_insnhelper.c // // For a general overview for how the interpreter calls methods, // see vm_call_method(). let ci = unsafe { get_call_data_ci(cd) }; // info about the call site let argc: i32 = unsafe { vm_ci_argc(ci) }.try_into().unwrap(); let mid = unsafe { vm_ci_mid(ci) }; let flags = unsafe { vm_ci_flag(ci) }; // Don't JIT calls with keyword splat if flags & VM_CALL_KW_SPLAT != 0 { gen_counter_incr!(asm, send_kw_splat); return CantCompile; } // Don't JIT calls that aren't simple // Note, not using VM_CALL_ARGS_SIMPLE because sometimes we pass a block. if flags & VM_CALL_ARGS_SPLAT != 0 { gen_counter_incr!(asm, send_args_splat); return CantCompile; } if flags & VM_CALL_ARGS_BLOCKARG != 0 { gen_counter_incr!(asm, send_block_arg); return CantCompile; } // Defer compilation so we can specialize on class of receiver if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let comptime_recv = jit_peek_at_stack(jit, ctx, argc as isize); let comptime_recv_klass = comptime_recv.class_of(); // Guard that the receiver has the same class as the one from compile time let side_exit = get_side_exit(jit, ocb, ctx); // Points to the receiver operand on the stack let recv = ctx.stack_opnd(argc); let recv_opnd = StackOpnd(argc.try_into().unwrap()); jit_guard_known_klass( jit, ctx, asm, ocb, comptime_recv_klass, recv, recv_opnd, comptime_recv, SEND_MAX_DEPTH, side_exit, ); // Do method lookup let mut cme = unsafe { rb_callable_method_entry(comptime_recv_klass, mid) }; if cme.is_null() { // TODO: counter return CantCompile; } let visi = unsafe { METHOD_ENTRY_VISI(cme) }; match visi { METHOD_VISI_PUBLIC => { // Can always call public methods } METHOD_VISI_PRIVATE => { if flags & VM_CALL_FCALL == 0 { // Can only call private methods with FCALL callsites. // (at the moment they are callsites without a receiver or an explicit `self` receiver) return CantCompile; } } METHOD_VISI_PROTECTED => { // If the method call is an FCALL, it is always valid if flags & VM_CALL_FCALL == 0 { // otherwise we need an ancestry check to ensure the receiver is vaild to be called // as protected jit_protected_callee_ancestry_guard(jit, asm, ocb, cme, side_exit); } } _ => { panic!("cmes should always have a visibility!"); } } // Register block for invalidation //assert!(cme->called_id == mid); assume_method_lookup_stable(jit, ocb, comptime_recv_klass, cme); // To handle the aliased method case (VM_METHOD_TYPE_ALIAS) loop { let def_type = unsafe { get_cme_def_type(cme) }; match def_type { VM_METHOD_TYPE_ISEQ => { return gen_send_iseq(jit, ctx, asm, ocb, ci, cme, block, argc); } VM_METHOD_TYPE_CFUNC => { return gen_send_cfunc( jit, ctx, asm, ocb, ci, cme, block, argc, &comptime_recv_klass, ); } VM_METHOD_TYPE_IVAR => { if argc != 0 { // Argument count mismatch. Getters take no arguments. gen_counter_incr!(asm, send_getter_arity); return CantCompile; } if c_method_tracing_currently_enabled(jit) { // Can't generate code for firing c_call and c_return events // :attr-tracing: // Handling the C method tracing events for attr_accessor // methods is easier than regular C methods as we know the // "method" we are calling into never enables those tracing // events. Once global invalidation runs, the code for the // attr_accessor is invalidated and we exit at the closest // instruction boundary which is always outside of the body of // the attr_accessor code. gen_counter_incr!(asm, send_cfunc_tracing); return CantCompile; } let ivar_name = unsafe { get_cme_def_body_attr_id(cme) }; return gen_get_ivar( jit, ctx, asm, ocb, SEND_MAX_DEPTH, comptime_recv, ivar_name, recv, recv_opnd, side_exit, ); } VM_METHOD_TYPE_ATTRSET => { if flags & VM_CALL_KWARG != 0 { gen_counter_incr!(asm, send_attrset_kwargs); return CantCompile; } else if argc != 1 || unsafe { !RB_TYPE_P(comptime_recv, RUBY_T_OBJECT) } { gen_counter_incr!(asm, send_ivar_set_method); return CantCompile; } else if c_method_tracing_currently_enabled(jit) { // Can't generate code for firing c_call and c_return events // See :attr-tracing: gen_counter_incr!(asm, send_cfunc_tracing); return CantCompile; } else { let ivar_name = unsafe { get_cme_def_body_attr_id(cme) }; return gen_set_ivar(jit, ctx, asm, comptime_recv, ivar_name); } } // Block method, e.g. define_method(:foo) { :my_block } VM_METHOD_TYPE_BMETHOD => { gen_counter_incr!(asm, send_bmethod); return CantCompile; } VM_METHOD_TYPE_ZSUPER => { gen_counter_incr!(asm, send_zsuper_method); return CantCompile; } VM_METHOD_TYPE_ALIAS => { // Retrieve the aliased method and re-enter the switch cme = unsafe { rb_aliased_callable_method_entry(cme) }; continue; } VM_METHOD_TYPE_UNDEF => { gen_counter_incr!(asm, send_undef_method); return CantCompile; } VM_METHOD_TYPE_NOTIMPLEMENTED => { gen_counter_incr!(asm, send_not_implemented_method); return CantCompile; } // Send family of methods, e.g. call/apply VM_METHOD_TYPE_OPTIMIZED => { let opt_type = unsafe { get_cme_def_body_optimized_type(cme) }; match opt_type { OPTIMIZED_METHOD_TYPE_SEND => { gen_counter_incr!(asm, send_optimized_method_send); return CantCompile; } OPTIMIZED_METHOD_TYPE_CALL => { gen_counter_incr!(asm, send_optimized_method_call); return CantCompile; } OPTIMIZED_METHOD_TYPE_BLOCK_CALL => { gen_counter_incr!(asm, send_optimized_method_block_call); return CantCompile; } OPTIMIZED_METHOD_TYPE_STRUCT_AREF => { return gen_struct_aref( jit, ctx, asm, ocb, ci, cme, comptime_recv, comptime_recv_klass, ); } OPTIMIZED_METHOD_TYPE_STRUCT_ASET => { return gen_struct_aset( jit, ctx, asm, ocb, ci, cme, comptime_recv, comptime_recv_klass, ); } _ => { panic!("unknown optimized method type!") } } } VM_METHOD_TYPE_MISSING => { gen_counter_incr!(asm, send_missing_method); return CantCompile; } VM_METHOD_TYPE_REFINED => { gen_counter_incr!(asm, send_refined_method); return CantCompile; } _ => { unreachable!(); } } } } fn gen_opt_send_without_block( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let cd = jit_get_arg(jit, 0).as_ptr(); gen_send_general(jit, ctx, asm, ocb, cd, None) } fn gen_send( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let cd = jit_get_arg(jit, 0).as_ptr(); let block = jit_get_arg(jit, 1).as_optional_ptr(); return gen_send_general(jit, ctx, asm, ocb, cd, block); } fn gen_invokesuper( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let cd: *const rb_call_data = jit_get_arg(jit, 0).as_ptr(); let block: Option = jit_get_arg(jit, 1).as_optional_ptr(); // Defer compilation so we can specialize on class of receiver if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let me = unsafe { rb_vm_frame_method_entry(get_ec_cfp(jit.ec.unwrap())) }; if me.is_null() { return CantCompile; } // FIXME: We should track and invalidate this block when this cme is invalidated let current_defined_class = unsafe { (*me).defined_class }; let mid = unsafe { get_def_original_id((*me).def) }; if me != unsafe { rb_callable_method_entry(current_defined_class, (*me).called_id) } { // Though we likely could generate this call, as we are only concerned // with the method entry remaining valid, assume_method_lookup_stable // below requires that the method lookup matches as well return CantCompile; } // vm_search_normal_superclass let rbasic_ptr: *const RBasic = current_defined_class.as_ptr(); if current_defined_class.builtin_type() == RUBY_T_ICLASS && unsafe { RB_TYPE_P((*rbasic_ptr).klass, RUBY_T_MODULE) && FL_TEST_RAW((*rbasic_ptr).klass, VALUE(RMODULE_IS_REFINEMENT.as_usize())) != VALUE(0) } { return CantCompile; } let comptime_superclass = unsafe { rb_class_get_superclass(RCLASS_ORIGIN(current_defined_class)) }; let ci = unsafe { get_call_data_ci(cd) }; let argc: i32 = unsafe { vm_ci_argc(ci) }.try_into().unwrap(); let ci_flags = unsafe { vm_ci_flag(ci) }; // Don't JIT calls that aren't simple // Note, not using VM_CALL_ARGS_SIMPLE because sometimes we pass a block. if ci_flags & VM_CALL_ARGS_SPLAT != 0 { gen_counter_incr!(asm, send_args_splat); return CantCompile; } if ci_flags & VM_CALL_KWARG != 0 { gen_counter_incr!(asm, send_keywords); return CantCompile; } if ci_flags & VM_CALL_KW_SPLAT != 0 { gen_counter_incr!(asm, send_kw_splat); return CantCompile; } if ci_flags & VM_CALL_ARGS_BLOCKARG != 0 { gen_counter_incr!(asm, send_block_arg); return CantCompile; } // Ensure we haven't rebound this method onto an incompatible class. // In the interpreter we try to avoid making this check by performing some // cheaper calculations first, but since we specialize on the method entry // and so only have to do this once at compile time this is fine to always // check and side exit. let comptime_recv = jit_peek_at_stack(jit, ctx, argc as isize); if unsafe { rb_obj_is_kind_of(comptime_recv, current_defined_class) } == VALUE(0) { return CantCompile; } // Do method lookup let cme = unsafe { rb_callable_method_entry(comptime_superclass, mid) }; if cme.is_null() { return CantCompile; } // Check that we'll be able to write this method dispatch before generating checks let cme_def_type = unsafe { get_cme_def_type(cme) }; if cme_def_type != VM_METHOD_TYPE_ISEQ && cme_def_type != VM_METHOD_TYPE_CFUNC { // others unimplemented return CantCompile; } // Guard that the receiver has the same class as the one from compile time let side_exit = get_side_exit(jit, ocb, ctx); let cfp = unsafe { get_ec_cfp(jit.ec.unwrap()) }; let ep = unsafe { get_cfp_ep(cfp) }; let cref_me = unsafe { *ep.offset(VM_ENV_DATA_INDEX_ME_CREF.try_into().unwrap()) }; let me_as_value = VALUE(me as usize); if cref_me != me_as_value { // This will be the case for super within a block return CantCompile; } asm.comment("guard known me"); let ep_opnd = asm.load(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP)); let ep_me_opnd = Opnd::mem( 64, ep_opnd, (SIZEOF_VALUE as i32) * (VM_ENV_DATA_INDEX_ME_CREF as i32), ); asm.cmp(ep_me_opnd, me_as_value.into()); asm.jne(counted_exit!(ocb, side_exit, invokesuper_me_changed).into()); if block.is_none() { // Guard no block passed // rb_vm_frame_block_handler(GET_EC()->cfp) == VM_BLOCK_HANDLER_NONE // note, we assume VM_ASSERT(VM_ENV_LOCAL_P(ep)) // // TODO: this could properly forward the current block handler, but // would require changes to gen_send_* asm.comment("guard no block given"); // EP is in REG0 from above let ep_opnd = asm.load(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP)); let ep_specval_opnd = Opnd::mem( 64, ep_opnd, (SIZEOF_VALUE as i32) * (VM_ENV_DATA_INDEX_SPECVAL as i32), ); asm.cmp(ep_specval_opnd, VM_BLOCK_HANDLER_NONE.into()); asm.jne(counted_exit!(ocb, side_exit, invokesuper_block).into()); } // We need to assume that both our current method entry and the super // method entry we invoke remain stable assume_method_lookup_stable(jit, ocb, current_defined_class, me); assume_method_lookup_stable(jit, ocb, comptime_superclass, cme); // Method calls may corrupt types ctx.clear_local_types(); match cme_def_type { VM_METHOD_TYPE_ISEQ => gen_send_iseq(jit, ctx, asm, ocb, ci, cme, block, argc), VM_METHOD_TYPE_CFUNC => { gen_send_cfunc(jit, ctx, asm, ocb, ci, cme, block, argc, ptr::null()) } _ => unreachable!(), } } fn gen_leave( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // Only the return value should be on the stack assert!(ctx.get_stack_size() == 1); // Create a side-exit to fall back to the interpreter let side_exit = get_side_exit(jit, ocb, ctx); let mut ocb_asm = Assembler::new(); // Check for interrupts gen_check_ints(asm, counted_exit!(ocb, side_exit, leave_se_interrupt)); ocb_asm.compile(ocb.unwrap()); // Pop the current frame (ec->cfp++) // Note: the return PC is already in the previous CFP asm.comment("pop stack frame"); let incr_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); asm.mov(CFP, incr_cfp); asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), incr_cfp); // Load the return value let retval_opnd = ctx.stack_pop(1); // Move the return value into the C return register for gen_leave_exit() asm.mov(C_RET_OPND, retval_opnd); // Reload REG_SP for the caller and write the return value. // Top of the stack is REG_SP[0] since the caller has sp_offset=1. asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); asm.mov(Opnd::mem(64, SP, 0), C_RET_OPND); // Jump to the JIT return address on the frame that was just popped let offset_to_jit_return = -(RUBY_SIZEOF_CONTROL_FRAME as i32) + (RUBY_OFFSET_CFP_JIT_RETURN as i32); asm.jmp_opnd(Opnd::mem(64, CFP, offset_to_jit_return)); EndBlock } fn gen_getglobal( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let gid = jit_get_arg(jit, 0); // Save the PC and SP because we might make a Ruby call for warning jit_prepare_routine_call(jit, ctx, asm); let val_opnd = asm.ccall( rb_gvar_get as *const u8, vec![ gid.into() ] ); let top = ctx.stack_push(Type::Unknown); asm.mov(top, val_opnd); KeepCompiling } fn gen_setglobal( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let gid = jit_get_arg(jit, 0); // Save the PC and SP because we might make a Ruby call for // Kernel#set_trace_var jit_prepare_routine_call(jit, ctx, asm); asm.ccall( rb_gvar_set as *const u8, vec![ gid.into(), ctx.stack_pop(1), ], ); KeepCompiling } fn gen_anytostring( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // Save the PC and SP since we might call #to_s jit_prepare_routine_call(jit, ctx, asm); let str = ctx.stack_pop(1); let val = ctx.stack_pop(1); let val = asm.ccall(rb_obj_as_string_result as *const u8, vec![str, val]); // Push the return value let stack_ret = ctx.stack_push(Type::TString); asm.mov(stack_ret, val); KeepCompiling } fn gen_objtostring( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let recv = ctx.stack_opnd(0); let comptime_recv = jit_peek_at_stack(jit, ctx, 0); if unsafe { RB_TYPE_P(comptime_recv, RUBY_T_STRING) } { let side_exit = get_side_exit(jit, ocb, ctx); jit_guard_known_klass( jit, ctx, asm, ocb, comptime_recv.class_of(), recv, StackOpnd(0), comptime_recv, SEND_MAX_DEPTH, side_exit, ); // No work needed. The string value is already on the top of the stack. KeepCompiling } else { let cd = jit_get_arg(jit, 0).as_ptr(); gen_send_general(jit, ctx, asm, ocb, cd, None) } } fn gen_intern( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // Save the PC and SP because we might allocate jit_prepare_routine_call(jit, ctx, asm); let str = ctx.stack_pop(1); let sym = asm.ccall(rb_str_intern as *const u8, vec![str]); // Push the return value let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, sym); KeepCompiling } fn gen_toregexp( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let opt = jit_get_arg(jit, 0).as_i64(); let cnt = jit_get_arg(jit, 1).as_usize(); // Save the PC and SP because this allocates an object and could // raise an exception. jit_prepare_routine_call(jit, ctx, asm); let values_ptr = asm.lea(ctx.sp_opnd(-((SIZEOF_VALUE as isize) * (cnt as isize)))); ctx.stack_pop(cnt); let ary = asm.ccall( rb_ary_tmp_new_from_values as *const u8, vec![ Opnd::Imm(0), Opnd::UImm(jit_get_arg(jit, 1).as_u64()), values_ptr, ] ); // Save the array so we can clear it later asm.cpush(ary); asm.cpush(ary); // Alignment let val = asm.ccall( rb_reg_new_ary as *const u8, vec![ ary, Opnd::Imm(opt), ] ); // The actual regex is in RAX now. Pop the temp array from // rb_ary_tmp_new_from_values into C arg regs so we can clear it let ary = asm.cpop(); // Alignment asm.cpop_into(ary); // The value we want to push on the stack is in RAX right now let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, val); // Clear the temp array. asm.ccall(rb_ary_clear as *const u8, vec![ary]); KeepCompiling } fn gen_getspecial( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // This takes two arguments, key and type // key is only used when type == 0 // A non-zero type determines which type of backref to fetch //rb_num_t key = jit_get_arg(jit, 0); let rtype = jit_get_arg(jit, 1).as_u64(); if rtype == 0 { // not yet implemented return CantCompile; } else if rtype & 0x01 != 0 { // Fetch a "special" backref based on a char encoded by shifting by 1 // Can raise if matchdata uninitialized jit_prepare_routine_call(jit, ctx, asm); // call rb_backref_get() asm.comment("rb_backref_get"); let backref = asm.ccall(rb_backref_get as *const u8, vec![]); let rt_u8: u8 = (rtype >> 1).try_into().unwrap(); let val = match rt_u8.into() { '&' => { asm.comment("rb_reg_last_match"); asm.ccall(rb_reg_last_match as *const u8, vec![backref]) } '`' => { asm.comment("rb_reg_match_pre"); asm.ccall(rb_reg_match_pre as *const u8, vec![backref]) } '\'' => { asm.comment("rb_reg_match_post"); asm.ccall(rb_reg_match_post as *const u8, vec![backref]) } '+' => { asm.comment("rb_reg_match_last"); asm.ccall(rb_reg_match_last as *const u8, vec![backref]) } _ => panic!("invalid back-ref"), }; let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, val); KeepCompiling } else { // Fetch the N-th match from the last backref based on type shifted by 1 // Can raise if matchdata uninitialized jit_prepare_routine_call(jit, ctx, asm); // call rb_backref_get() asm.comment("rb_backref_get"); let backref = asm.ccall(rb_backref_get as *const u8, vec![]); // rb_reg_nth_match((int)(type >> 1), backref); asm.comment("rb_reg_nth_match"); let val = asm.ccall( rb_reg_nth_match as *const u8, vec![ Opnd::Imm((rtype >> 1).try_into().unwrap()), backref, ] ); let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, val); KeepCompiling } } fn gen_getclassvariable( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // rb_vm_getclassvariable can raise exceptions. jit_prepare_routine_call(jit, ctx, asm); let val_opnd = asm.ccall( rb_vm_getclassvariable as *const u8, vec![ Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), CFP, Opnd::UImm(jit_get_arg(jit, 0).as_u64()), Opnd::UImm(jit_get_arg(jit, 1).as_u64()), ], ); let top = ctx.stack_push(Type::Unknown); asm.mov(top, val_opnd); KeepCompiling } fn gen_setclassvariable( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { // rb_vm_setclassvariable can raise exceptions. jit_prepare_routine_call(jit, ctx, asm); asm.ccall( rb_vm_setclassvariable as *const u8, vec![ Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), CFP, Opnd::UImm(jit_get_arg(jit, 0).as_u64()), ctx.stack_pop(1), Opnd::UImm(jit_get_arg(jit, 1).as_u64()), ], ); KeepCompiling } fn gen_opt_getinlinecache( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { let jump_offset = jit_get_arg(jit, 0); let const_cache_as_value = jit_get_arg(jit, 1); let ic: *const iseq_inline_constant_cache = const_cache_as_value.as_ptr(); // See vm_ic_hit_p(). The same conditions are checked in yjit_constant_ic_update(). let ice = unsafe { (*ic).entry }; if ice.is_null() { // In this case, leave a block that unconditionally side exits // for the interpreter to invalidate. return CantCompile; } // Make sure there is an exit for this block as the interpreter might want // to invalidate this block from yjit_constant_ic_update(). jit_ensure_block_entry_exit(jit, ocb); if !unsafe { (*ice).ic_cref }.is_null() { // Cache is keyed on a certain lexical scope. Use the interpreter's cache. let side_exit = get_side_exit(jit, ocb, ctx); let inline_cache = asm.load(Opnd::const_ptr(ic as *const u8)); // Call function to verify the cache. It doesn't allocate or call methods. let ret_val = asm.ccall( rb_vm_ic_hit_p as *const u8, vec![inline_cache, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP)] ); // Check the result. _Bool is one byte in SysV. asm.test(ret_val, ret_val); asm.jz(counted_exit!(ocb, side_exit, opt_getinlinecache_miss).into()); let inline_cache = asm.load(Opnd::const_ptr(ic as *const u8)); let ic_entry = asm.load(Opnd::mem( 64, inline_cache, RUBY_OFFSET_IC_ENTRY )); let ic_entry_val = asm.load(Opnd::mem( 64, ic_entry, RUBY_OFFSET_ICE_VALUE )); // Push ic->entry->value let stack_top = ctx.stack_push(Type::Unknown); asm.store(stack_top, ic_entry_val); } else { // Optimize for single ractor mode. // FIXME: This leaks when st_insert raises NoMemoryError if !assume_single_ractor_mode(jit, ocb) { return CantCompile; } // Invalidate output code on any constant writes associated with // constants referenced within the current block. assume_stable_constant_names(jit, ocb); jit_putobject(jit, ctx, asm, unsafe { (*ice).value }); } // Jump over the code for filling the cache let jump_idx = jit_next_insn_idx(jit) + jump_offset.as_u32(); gen_direct_jump( jit, ctx, BlockId { iseq: jit.iseq, idx: jump_idx, }, asm, ); EndBlock } // Push the explicit block parameter onto the temporary stack. Part of the // interpreter's scheme for avoiding Proc allocations when delegating // explicit block parameters. fn gen_getblockparamproxy( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { if !jit_at_current_insn(jit) { defer_compilation(jit, ctx, asm, ocb); return EndBlock; } let starting_context = *ctx; // make a copy for use with jit_chain_guard // A mirror of the interpreter code. Checking for the case // where it's pushing rb_block_param_proxy. let side_exit = get_side_exit(jit, ocb, ctx); // EP level let level = jit_get_arg(jit, 1).as_u32(); // Peek at the block handler so we can check whether it's nil let comptime_handler = jit_peek_at_block_handler(jit, level); // When a block handler is present, it should always be a GC-guarded // pointer (VM_BH_ISEQ_BLOCK_P) if comptime_handler.as_u64() != 0 && comptime_handler.as_u64() & 0x3 != 0x1 { return CantCompile; } // Load environment pointer EP from CFP let ep_opnd = gen_get_ep(asm, level); // Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero let flag_check = Opnd::mem( 64, ep_opnd, (SIZEOF_VALUE as i32) * (VM_ENV_DATA_INDEX_FLAGS as i32), ); asm.test(flag_check, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); asm.jnz(counted_exit!(ocb, side_exit, gbpp_block_param_modified).into()); // Load the block handler for the current frame // note, VM_ASSERT(VM_ENV_LOCAL_P(ep)) let block_handler = asm.load( Opnd::mem(64, ep_opnd, (SIZEOF_VALUE as i32) * (VM_ENV_DATA_INDEX_SPECVAL as i32)) ); // Specialize compilation for the case where no block handler is present if comptime_handler.as_u64() == 0 { // Bail if there is a block handler asm.cmp(block_handler, Opnd::UImm(0)); jit_chain_guard( JCC_JNZ, jit, &starting_context, asm, ocb, SEND_MAX_DEPTH, side_exit, ); jit_putobject(jit, ctx, asm, Qnil); } else { // Block handler is a tagged pointer. Look at the tag. 0x03 is from VM_BH_ISEQ_BLOCK_P(). let block_handler = asm.and(block_handler, 0x3.into()); // Bail unless VM_BH_ISEQ_BLOCK_P(bh). This also checks for null. asm.cmp(block_handler, 0x1.into()); jit_chain_guard( JCC_JNZ, jit, &starting_context, asm, ocb, SEND_MAX_DEPTH, side_exit, ); // Push rb_block_param_proxy. It's a root, so no need to use jit_mov_gc_ptr. assert!(!unsafe { rb_block_param_proxy }.special_const_p()); let top = ctx.stack_push(Type::Unknown); asm.mov(top, Opnd::const_ptr(unsafe { rb_block_param_proxy }.as_ptr())); } jump_to_next_insn(jit, ctx, asm, ocb); EndBlock } fn gen_getblockparam( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ) -> CodegenStatus { // EP level let level = jit_get_arg(jit, 1).as_u32(); // Save the PC and SP because we might allocate jit_prepare_routine_call(jit, ctx, asm); // A mirror of the interpreter code. Checking for the case // where it's pushing rb_block_param_proxy. let side_exit = get_side_exit(jit, ocb, ctx); // Load environment pointer EP from CFP let ep_opnd = gen_get_ep(asm, level); // Bail when VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM) is non zero let flag_check = Opnd::mem(64, ep_opnd, (SIZEOF_VALUE as i32) * (VM_ENV_DATA_INDEX_FLAGS as i32)); // FIXME: This is testing bits in the same place that the WB check is testing. // We should combine these at some point asm.test(flag_check, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); // If the frame flag has been modified, then the actual proc value is // already in the EP and we should just use the value. let frame_flag_modified = asm.new_label("frame_flag_modified"); asm.jnz(frame_flag_modified); // This instruction writes the block handler to the EP. If we need to // fire a write barrier for the write, then exit (we'll let the // interpreter handle it so it can fire the write barrier). // flags & VM_ENV_FLAG_WB_REQUIRED let flags_opnd = Opnd::mem( 64, ep_opnd, SIZEOF_VALUE as i32 * VM_ENV_DATA_INDEX_FLAGS as i32, ); asm.test(flags_opnd, VM_ENV_FLAG_WB_REQUIRED.into()); // if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0 asm.jnz(side_exit.into()); // Convert the block handler in to a proc // call rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler) let proc = asm.ccall( rb_vm_bh_to_procval as *const u8, vec![ EC, // The block handler for the current frame // note, VM_ASSERT(VM_ENV_LOCAL_P(ep)) Opnd::mem( 64, ep_opnd, (SIZEOF_VALUE as i32) * (VM_ENV_DATA_INDEX_SPECVAL as i32), ), ] ); // Load environment pointer EP from CFP (again) let ep_opnd = gen_get_ep(asm, level); // Write the value at the environment pointer let idx = jit_get_arg(jit, 0).as_i32(); let offs = -(SIZEOF_VALUE as i32 * idx); asm.mov(Opnd::mem(64, ep_opnd, offs), proc); // Set the frame modified flag let flag_check = Opnd::mem(64, ep_opnd, (SIZEOF_VALUE as i32) * (VM_ENV_DATA_INDEX_FLAGS as i32)); let modified_flag = asm.or(flag_check, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); asm.store(flag_check, modified_flag); asm.write_label(frame_flag_modified); // Push the proc on the stack let stack_ret = ctx.stack_push(Type::Unknown); let ep_opnd = gen_get_ep(asm, level); asm.mov(stack_ret, Opnd::mem(64, ep_opnd, offs)); KeepCompiling } fn gen_invokebuiltin( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let bf: *const rb_builtin_function = jit_get_arg(jit, 0).as_ptr(); let bf_argc: usize = unsafe { (*bf).argc }.try_into().expect("non negative argc"); // ec, self, and arguments if bf_argc + 2 > C_ARG_OPNDS.len() { return CantCompile; } // If the calls don't allocate, do they need up to date PC, SP? jit_prepare_routine_call(jit, ctx, asm); // Call the builtin func (ec, recv, arg1, arg2, ...) let mut args = vec![EC, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)]; // Copy arguments from locals for i in 0..bf_argc { let stack_opnd = ctx.stack_opnd((bf_argc - i - 1) as i32); args.push(stack_opnd); } let val = asm.ccall(unsafe { (*bf).func_ptr } as *const u8, args); // Push the return value ctx.stack_pop(bf_argc); let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, val); KeepCompiling } // opt_invokebuiltin_delegate calls a builtin function, like // invokebuiltin does, but instead of taking arguments from the top of the // stack uses the argument locals (and self) from the current method. fn gen_opt_invokebuiltin_delegate( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, _ocb: &mut OutlinedCb, ) -> CodegenStatus { let bf: *const rb_builtin_function = jit_get_arg(jit, 0).as_ptr(); let bf_argc = unsafe { (*bf).argc }; let start_index = jit_get_arg(jit, 1).as_i32(); // ec, self, and arguments if bf_argc + 2 > (C_ARG_OPNDS.len() as i32) { return CantCompile; } // If the calls don't allocate, do they need up to date PC, SP? jit_prepare_routine_call(jit, ctx, asm); // Call the builtin func (ec, recv, arg1, arg2, ...) let mut args = vec![EC, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)]; // Copy arguments from locals if bf_argc > 0 { // Load environment pointer EP from CFP let ep = asm.load(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP)); for i in 0..bf_argc { let table_size = unsafe { get_iseq_body_local_table_size(jit.iseq) }; let offs: i32 = -(table_size as i32) - (VM_ENV_DATA_SIZE as i32) + 1 + start_index + i; let local_opnd = Opnd::mem(64, ep, offs * (SIZEOF_VALUE as i32)); args.push(local_opnd); } } let val = asm.ccall(unsafe { (*bf).func_ptr } as *const u8, args); // Push the return value let stack_ret = ctx.stack_push(Type::Unknown); asm.mov(stack_ret, val); KeepCompiling } /// Maps a YARV opcode to a code generation function (if supported) fn get_gen_fn(opcode: VALUE) -> Option { let VALUE(opcode) = opcode; let opcode = opcode as ruby_vminsn_type; assert!(opcode < VM_INSTRUCTION_SIZE); match opcode { YARVINSN_nop => Some(gen_nop), YARVINSN_pop => Some(gen_pop), YARVINSN_dup => Some(gen_dup), YARVINSN_dupn => Some(gen_dupn), YARVINSN_swap => Some(gen_swap), YARVINSN_putnil => Some(gen_putnil), YARVINSN_putobject => Some(gen_putobject), YARVINSN_putobject_INT2FIX_0_ => Some(gen_putobject_int2fix), YARVINSN_putobject_INT2FIX_1_ => Some(gen_putobject_int2fix), YARVINSN_putself => Some(gen_putself), YARVINSN_putspecialobject => Some(gen_putspecialobject), YARVINSN_setn => Some(gen_setn), YARVINSN_topn => Some(gen_topn), YARVINSN_adjuststack => Some(gen_adjuststack), YARVINSN_getlocal => Some(gen_getlocal), YARVINSN_getlocal_WC_0 => Some(gen_getlocal_wc0), YARVINSN_getlocal_WC_1 => Some(gen_getlocal_wc1), YARVINSN_setlocal => Some(gen_setlocal), YARVINSN_setlocal_WC_0 => Some(gen_setlocal_wc0), YARVINSN_setlocal_WC_1 => Some(gen_setlocal_wc1), YARVINSN_opt_plus => Some(gen_opt_plus), YARVINSN_opt_minus => Some(gen_opt_minus), YARVINSN_opt_and => Some(gen_opt_and), YARVINSN_opt_or => Some(gen_opt_or), YARVINSN_newhash => Some(gen_newhash), YARVINSN_duphash => Some(gen_duphash), YARVINSN_newarray => Some(gen_newarray), YARVINSN_duparray => Some(gen_duparray), YARVINSN_checktype => Some(gen_checktype), YARVINSN_opt_lt => Some(gen_opt_lt), YARVINSN_opt_le => Some(gen_opt_le), YARVINSN_opt_gt => Some(gen_opt_gt), YARVINSN_opt_ge => Some(gen_opt_ge), YARVINSN_opt_mod => Some(gen_opt_mod), YARVINSN_opt_str_freeze => Some(gen_opt_str_freeze), YARVINSN_opt_str_uminus => Some(gen_opt_str_uminus), YARVINSN_splatarray => Some(gen_splatarray), YARVINSN_concatarray => Some(gen_concatarray), YARVINSN_newrange => Some(gen_newrange), YARVINSN_putstring => Some(gen_putstring), YARVINSN_expandarray => Some(gen_expandarray), YARVINSN_defined => Some(gen_defined), YARVINSN_checkkeyword => Some(gen_checkkeyword), YARVINSN_concatstrings => Some(gen_concatstrings), YARVINSN_getinstancevariable => Some(gen_getinstancevariable), YARVINSN_setinstancevariable => Some(gen_setinstancevariable), YARVINSN_opt_eq => Some(gen_opt_eq), YARVINSN_opt_neq => Some(gen_opt_neq), YARVINSN_opt_aref => Some(gen_opt_aref), YARVINSN_opt_aset => Some(gen_opt_aset), YARVINSN_opt_mult => Some(gen_opt_mult), YARVINSN_opt_div => Some(gen_opt_div), YARVINSN_opt_ltlt => Some(gen_opt_ltlt), YARVINSN_opt_nil_p => Some(gen_opt_nil_p), YARVINSN_opt_empty_p => Some(gen_opt_empty_p), YARVINSN_opt_succ => Some(gen_opt_succ), YARVINSN_opt_not => Some(gen_opt_not), YARVINSN_opt_size => Some(gen_opt_size), YARVINSN_opt_length => Some(gen_opt_length), YARVINSN_opt_regexpmatch2 => Some(gen_opt_regexpmatch2), YARVINSN_opt_getinlinecache => Some(gen_opt_getinlinecache), YARVINSN_invokebuiltin => Some(gen_invokebuiltin), YARVINSN_opt_invokebuiltin_delegate => Some(gen_opt_invokebuiltin_delegate), YARVINSN_opt_invokebuiltin_delegate_leave => Some(gen_opt_invokebuiltin_delegate), YARVINSN_opt_case_dispatch => Some(gen_opt_case_dispatch), YARVINSN_branchif => Some(gen_branchif), YARVINSN_branchunless => Some(gen_branchunless), YARVINSN_branchnil => Some(gen_branchnil), YARVINSN_jump => Some(gen_jump), YARVINSN_getblockparamproxy => Some(gen_getblockparamproxy), YARVINSN_getblockparam => Some(gen_getblockparam), YARVINSN_opt_send_without_block => Some(gen_opt_send_without_block), YARVINSN_send => Some(gen_send), YARVINSN_invokesuper => Some(gen_invokesuper), YARVINSN_leave => Some(gen_leave), YARVINSN_getglobal => Some(gen_getglobal), YARVINSN_setglobal => Some(gen_setglobal), YARVINSN_anytostring => Some(gen_anytostring), YARVINSN_objtostring => Some(gen_objtostring), YARVINSN_intern => Some(gen_intern), YARVINSN_toregexp => Some(gen_toregexp), YARVINSN_getspecial => Some(gen_getspecial), YARVINSN_getclassvariable => Some(gen_getclassvariable), YARVINSN_setclassvariable => Some(gen_setclassvariable), // Unimplemented opcode, YJIT won't generate code for this yet _ => None, } } // Return true when the codegen function generates code. // known_recv_klass is non-NULL when the caller has used jit_guard_known_klass(). // See yjit_reg_method(). type MethodGenFn = fn( jit: &mut JITState, ctx: &mut Context, asm: &mut Assembler, ocb: &mut OutlinedCb, ci: *const rb_callinfo, cme: *const rb_callable_method_entry_t, block: Option, argc: i32, known_recv_class: *const VALUE, ) -> bool; /// Global state needed for code generation pub struct CodegenGlobals { /// Inline code block (fast path) inline_cb: CodeBlock, /// Outlined code block (slow path) outlined_cb: OutlinedCb, /// Code for exiting back to the interpreter from the leave instruction leave_exit_code: CodePtr, // For exiting from YJIT frame from branch_stub_hit(). // Filled by gen_code_for_exit_from_stub(). stub_exit_code: CodePtr, // Code for full logic of returning from C method and exiting to the interpreter outline_full_cfunc_return_pos: CodePtr, /// For implementing global code invalidation global_inval_patches: Vec, /// For implementing global code invalidation. The number of bytes counting from the beginning /// of the inline code block that should not be changed. After patching for global invalidation, /// no one should make changes to the invalidated code region anymore. This is used to /// break out of invalidation race when there are multiple ractors. inline_frozen_bytes: usize, // Methods for generating code for hardcoded (usually C) methods method_codegen_table: HashMap, } /// For implementing global code invalidation. A position in the inline /// codeblock to patch into a JMP rel32 which jumps into some code in /// the outlined codeblock to exit to the interpreter. pub struct CodepagePatch { pub inline_patch_pos: CodePtr, pub outlined_target_pos: CodePtr, } /// Private singleton instance of the codegen globals static mut CODEGEN_GLOBALS: Option = None; impl CodegenGlobals { /// Initialize the codegen globals pub fn init() { // Executable memory size in MiB let mem_size = get_option!(exec_mem_size) * 1024 * 1024; #[cfg(not(test))] let (mut cb, mut ocb) = { // TODO(alan): we can error more gracefully when the user gives // --yjit-exec-mem=absurdly-large-number // // 2 GiB. It's likely a bug if we generate this much code. const MAX_BUFFER_SIZE: usize = 2 * 1024 * 1024 * 1024; assert!(mem_size <= MAX_BUFFER_SIZE); let mem_size_u32 = mem_size as u32; let half_size = mem_size / 2; let page_size = unsafe { rb_yjit_get_page_size() }; let assert_page_aligned = |ptr| assert_eq!( 0, ptr as usize % page_size.as_usize(), "Start of virtual address block should be page-aligned", ); let virt_block: *mut u8 = unsafe { rb_yjit_reserve_addr_space(mem_size_u32) }; let second_half = virt_block.wrapping_add(half_size); // Memory protection syscalls need page-aligned addresses, so check it here. Assuming // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the // page size in bytes is a power of two 2¹⁹ or smaller. This is because the user // requested size is half of mem_option × 2²⁰ as it's in MiB. // // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. assert_page_aligned(virt_block); assert_page_aligned(second_half); use crate::virtualmem::*; let first_half = VirtualMem::new( SystemAllocator {}, page_size, virt_block, half_size ); let second_half = VirtualMem::new( SystemAllocator {}, page_size, second_half, half_size ); let cb = CodeBlock::new(first_half); let ocb = OutlinedCb::wrap(CodeBlock::new(second_half)); (cb, ocb) }; // In test mode we're not linking with the C code // so we don't allocate executable memory #[cfg(test)] let mut cb = CodeBlock::new_dummy(mem_size / 2); #[cfg(test)] let mut ocb = OutlinedCb::wrap(CodeBlock::new_dummy(mem_size / 2)); let leave_exit_code = gen_leave_exit(&mut ocb); let stub_exit_code = gen_code_for_exit_from_stub(&mut ocb); // Generate full exit code for C func let cfunc_exit_code = gen_full_cfunc_return(&mut ocb); // Mark all code memory as executable cb.mark_all_executable(); ocb.unwrap().mark_all_executable(); let mut codegen_globals = CodegenGlobals { inline_cb: cb, outlined_cb: ocb, leave_exit_code, stub_exit_code: stub_exit_code, outline_full_cfunc_return_pos: cfunc_exit_code, global_inval_patches: Vec::new(), inline_frozen_bytes: 0, method_codegen_table: HashMap::new(), }; // Register the method codegen functions codegen_globals.reg_method_codegen_fns(); // Initialize the codegen globals instance unsafe { CODEGEN_GLOBALS = Some(codegen_globals); } } // Register a specialized codegen function for a particular method. Note that // the if the function returns true, the code it generates runs without a // control frame and without interrupt checks. To avoid creating observable // behavior changes, the codegen function should only target simple code paths // that do not allocate and do not make method calls. fn yjit_reg_method(&mut self, klass: VALUE, mid_str: &str, gen_fn: MethodGenFn) { let id_string = std::ffi::CString::new(mid_str).expect("couldn't convert to CString!"); let mid = unsafe { rb_intern(id_string.as_ptr()) }; let me = unsafe { rb_method_entry_at(klass, mid) }; if me.is_null() { panic!("undefined optimized method!"); } // For now, only cfuncs are supported //RUBY_ASSERT(me && me->def); //RUBY_ASSERT(me->def->type == VM_METHOD_TYPE_CFUNC); let method_serial = unsafe { let def = (*me).def; get_def_method_serial(def) }; self.method_codegen_table.insert(method_serial, gen_fn); } /// Register codegen functions for some Ruby core methods fn reg_method_codegen_fns(&mut self) { unsafe { // Specialization for C methods. See yjit_reg_method() for details. self.yjit_reg_method(rb_cBasicObject, "!", jit_rb_obj_not); self.yjit_reg_method(rb_cNilClass, "nil?", jit_rb_true); self.yjit_reg_method(rb_mKernel, "nil?", jit_rb_false); self.yjit_reg_method(rb_cBasicObject, "==", jit_rb_obj_equal); self.yjit_reg_method(rb_cBasicObject, "equal?", jit_rb_obj_equal); self.yjit_reg_method(rb_mKernel, "eql?", jit_rb_obj_equal); self.yjit_reg_method(rb_cModule, "==", jit_rb_obj_equal); self.yjit_reg_method(rb_cSymbol, "==", jit_rb_obj_equal); self.yjit_reg_method(rb_cSymbol, "===", jit_rb_obj_equal); // rb_str_to_s() methods in string.c //self.yjit_reg_method(rb_cString, "to_s", jit_rb_str_to_s); //self.yjit_reg_method(rb_cString, "to_str", jit_rb_str_to_s); self.yjit_reg_method(rb_cString, "bytesize", jit_rb_str_bytesize); //self.yjit_reg_method(rb_cString, "<<", jit_rb_str_concat); self.yjit_reg_method(rb_cString, "+@", jit_rb_str_uplus); // Thread.current self.yjit_reg_method( rb_singleton_class(rb_cThread), "current", jit_thread_s_current, ); } } /// Get a mutable reference to the codegen globals instance pub fn get_instance() -> &'static mut CodegenGlobals { unsafe { CODEGEN_GLOBALS.as_mut().unwrap() } } /// Get a mutable reference to the inline code block pub fn get_inline_cb() -> &'static mut CodeBlock { &mut CodegenGlobals::get_instance().inline_cb } /// Get a mutable reference to the outlined code block pub fn get_outlined_cb() -> &'static mut OutlinedCb { &mut CodegenGlobals::get_instance().outlined_cb } pub fn get_leave_exit_code() -> CodePtr { CodegenGlobals::get_instance().leave_exit_code } pub fn get_stub_exit_code() -> CodePtr { CodegenGlobals::get_instance().stub_exit_code } pub fn push_global_inval_patch(i_pos: CodePtr, o_pos: CodePtr) { let patch = CodepagePatch { inline_patch_pos: i_pos, outlined_target_pos: o_pos, }; CodegenGlobals::get_instance() .global_inval_patches .push(patch); } // Drain the list of patches and return it pub fn take_global_inval_patches() -> Vec { let globals = CodegenGlobals::get_instance(); mem::take(&mut globals.global_inval_patches) } pub fn get_inline_frozen_bytes() -> usize { CodegenGlobals::get_instance().inline_frozen_bytes } pub fn set_inline_frozen_bytes(frozen_bytes: usize) { CodegenGlobals::get_instance().inline_frozen_bytes = frozen_bytes; } pub fn get_outline_full_cfunc_return_pos() -> CodePtr { CodegenGlobals::get_instance().outline_full_cfunc_return_pos } pub fn look_up_codegen_method(method_serial: usize) -> Option { let table = &CodegenGlobals::get_instance().method_codegen_table; let option_ref = table.get(&method_serial); match option_ref { None => None, Some(&mgf) => Some(mgf), // Deref } } } #[cfg(test)] mod tests { use super::*; fn setup_codegen() -> (JITState, Context, Assembler, CodeBlock, OutlinedCb) { let blockid = BlockId { iseq: ptr::null(), idx: 0, }; let block = Block::new(blockid, &Context::default()); return ( JITState::new(&block), Context::new(), Assembler::new(), CodeBlock::new_dummy(256 * 1024), OutlinedCb::wrap(CodeBlock::new_dummy(256 * 1024)), ); } #[test] fn test_gen_leave_exit() { let mut ocb = OutlinedCb::wrap(CodeBlock::new_dummy(256 * 1024)); gen_leave_exit(&mut ocb); assert!(ocb.unwrap().get_write_pos() > 0); } #[test] fn test_gen_exit() { let (_, ctx, mut asm, mut cb, _) = setup_codegen(); gen_exit(0 as *mut VALUE, &ctx, &mut asm); asm.compile(&mut cb); assert!(cb.get_write_pos() > 0); } #[test] fn test_get_side_exit() { let (mut jit, ctx, _, _, mut ocb) = setup_codegen(); get_side_exit(&mut jit, &mut ocb, &ctx); assert!(ocb.unwrap().get_write_pos() > 0); } #[test] fn test_gen_check_ints() { let (_, _ctx, mut asm, mut cb, mut ocb) = setup_codegen(); let side_exit = ocb.unwrap().get_write_ptr(); gen_check_ints(&mut asm, side_exit); } #[test] fn test_gen_nop() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); let status = gen_nop(&mut jit, &mut context, &mut asm, &mut ocb); asm.compile(&mut cb); assert_eq!(status, KeepCompiling); assert_eq!(context.diff(&Context::new()), 0); assert_eq!(cb.get_write_pos(), 0); } #[test] fn test_gen_pop() { let (mut jit, _, mut asm, mut cb, mut ocb) = setup_codegen(); let mut context = Context::new_with_stack_size(1); let status = gen_pop(&mut jit, &mut context, &mut asm, &mut ocb); assert_eq!(status, KeepCompiling); assert_eq!(context.diff(&Context::new()), 0); } #[test] fn test_gen_dup() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); context.stack_push(Type::Fixnum); let status = gen_dup(&mut jit, &mut context, &mut asm, &mut ocb); assert_eq!(status, KeepCompiling); // Did we duplicate the type information for the Fixnum type? assert_eq!(Type::Fixnum, context.get_opnd_type(StackOpnd(0))); assert_eq!(Type::Fixnum, context.get_opnd_type(StackOpnd(1))); asm.compile(&mut cb); assert!(cb.get_write_pos() > 0); // Write some movs } #[test] fn test_gen_dupn() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); context.stack_push(Type::Fixnum); context.stack_push(Type::Flonum); let mut value_array: [u64; 2] = [0, 2]; // We only compile for n == 2 let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; let status = gen_dupn(&mut jit, &mut context, &mut asm, &mut ocb); assert_eq!(status, KeepCompiling); assert_eq!(Type::Fixnum, context.get_opnd_type(StackOpnd(3))); assert_eq!(Type::Flonum, context.get_opnd_type(StackOpnd(2))); assert_eq!(Type::Fixnum, context.get_opnd_type(StackOpnd(1))); assert_eq!(Type::Flonum, context.get_opnd_type(StackOpnd(0))); // TODO: this is writing zero bytes on x86. Why? asm.compile(&mut cb); assert!(cb.get_write_pos() > 0); // Write some movs } #[test] fn test_gen_swap() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); context.stack_push(Type::Fixnum); context.stack_push(Type::Flonum); let status = gen_swap(&mut jit, &mut context, &mut asm, &mut ocb); let (_, tmp_type_top) = context.get_opnd_mapping(StackOpnd(0)); let (_, tmp_type_next) = context.get_opnd_mapping(StackOpnd(1)); assert_eq!(status, KeepCompiling); assert_eq!(tmp_type_top, Type::Fixnum); assert_eq!(tmp_type_next, Type::Flonum); } #[test] fn test_putnil() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); let status = gen_putnil(&mut jit, &mut context, &mut asm, &mut ocb); let (_, tmp_type_top) = context.get_opnd_mapping(StackOpnd(0)); assert_eq!(status, KeepCompiling); assert_eq!(tmp_type_top, Type::Nil); asm.compile(&mut cb); assert!(cb.get_write_pos() > 0); } #[test] fn test_putobject_qtrue() { // Test gen_putobject with Qtrue let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); let mut value_array: [u64; 2] = [0, Qtrue.into()]; let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; let status = gen_putobject(&mut jit, &mut context, &mut asm, &mut ocb); let (_, tmp_type_top) = context.get_opnd_mapping(StackOpnd(0)); assert_eq!(status, KeepCompiling); assert_eq!(tmp_type_top, Type::True); asm.compile(&mut cb); assert!(cb.get_write_pos() > 0); } #[test] fn test_putobject_fixnum() { // Test gen_putobject with a Fixnum to test another conditional branch let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); // The Fixnum 7 is encoded as 7 * 2 + 1, or 15 let mut value_array: [u64; 2] = [0, 15]; let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; let status = gen_putobject(&mut jit, &mut context, &mut asm, &mut ocb); let (_, tmp_type_top) = context.get_opnd_mapping(StackOpnd(0)); assert_eq!(status, KeepCompiling); assert_eq!(tmp_type_top, Type::Fixnum); asm.compile(&mut cb); assert!(cb.get_write_pos() > 0); } #[test] fn test_int2fix() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); jit.opcode = YARVINSN_putobject_INT2FIX_0_.as_usize(); let status = gen_putobject_int2fix(&mut jit, &mut context, &mut asm, &mut ocb); let (_, tmp_type_top) = context.get_opnd_mapping(StackOpnd(0)); // Right now we're not testing the generated machine code to make sure a literal 1 or 0 was pushed. I've checked locally. assert_eq!(status, KeepCompiling); assert_eq!(tmp_type_top, Type::Fixnum); } #[test] fn test_putself() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); let status = gen_putself(&mut jit, &mut context, &mut asm, &mut ocb); assert_eq!(status, KeepCompiling); asm.compile(&mut cb); assert!(cb.get_write_pos() > 0); } #[test] fn test_gen_setn() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); context.stack_push(Type::Fixnum); context.stack_push(Type::Flonum); context.stack_push(Type::CString); let mut value_array: [u64; 2] = [0, 2]; let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; let status = gen_setn(&mut jit, &mut context, &mut asm, &mut ocb); assert_eq!(status, KeepCompiling); assert_eq!(Type::CString, context.get_opnd_type(StackOpnd(2))); assert_eq!(Type::Flonum, context.get_opnd_type(StackOpnd(1))); assert_eq!(Type::CString, context.get_opnd_type(StackOpnd(0))); asm.compile(&mut cb); assert!(cb.get_write_pos() > 0); } #[test] fn test_gen_topn() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); context.stack_push(Type::Flonum); context.stack_push(Type::CString); let mut value_array: [u64; 2] = [0, 1]; let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; let status = gen_topn(&mut jit, &mut context, &mut asm, &mut ocb); assert_eq!(status, KeepCompiling); assert_eq!(Type::Flonum, context.get_opnd_type(StackOpnd(2))); assert_eq!(Type::CString, context.get_opnd_type(StackOpnd(1))); assert_eq!(Type::Flonum, context.get_opnd_type(StackOpnd(0))); asm.compile(&mut cb); assert!(cb.get_write_pos() > 0); // Write some movs } #[test] fn test_gen_adjuststack() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); context.stack_push(Type::Flonum); context.stack_push(Type::CString); context.stack_push(Type::Fixnum); let mut value_array: [u64; 3] = [0, 2, 0]; let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; jit.pc = pc; let status = gen_adjuststack(&mut jit, &mut context, &mut asm, &mut ocb); assert_eq!(status, KeepCompiling); assert_eq!(Type::Flonum, context.get_opnd_type(StackOpnd(0))); asm.compile(&mut cb); assert!(cb.get_write_pos() == 0); // No instructions written } #[test] fn test_gen_leave() { let (mut jit, mut context, mut asm, mut cb, mut ocb) = setup_codegen(); // Push return value context.stack_push(Type::Fixnum); gen_leave(&mut jit, &mut context, &mut asm, &mut ocb); } }