summaryrefslogtreecommitdiff
path: root/yjit/src/codegen.rs
diff options
context:
space:
mode:
Diffstat (limited to 'yjit/src/codegen.rs')
-rw-r--r--yjit/src/codegen.rs319
1 files changed, 271 insertions, 48 deletions
diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs
index 8ab5e0e230..d86f0d1955 100644
--- a/yjit/src/codegen.rs
+++ b/yjit/src/codegen.rs
@@ -46,7 +46,7 @@ type InsnGenFn = fn(
/// Represents a [core::Block] while we build it.
pub struct JITState {
/// Instruction sequence for the compiling block
- iseq: IseqPtr,
+ pub iseq: IseqPtr,
/// The iseq index of the first instruction in the block
starting_insn_idx: IseqIdx,
@@ -101,6 +101,9 @@ pub struct JITState {
/// A list of classes that are not supposed to have a singleton class.
pub no_singleton_class_assumptions: Vec<VALUE>,
+ /// When true, the block is valid only when base pointer is equal to environment pointer.
+ pub no_ep_escape: bool,
+
/// When true, the block is valid only when there is a total of one ractor running
pub block_assumes_single_ractor: bool,
@@ -130,6 +133,7 @@ impl JITState {
bop_assumptions: vec![],
stable_constant_names_assumption: None,
no_singleton_class_assumptions: vec![],
+ no_ep_escape: false,
block_assumes_single_ractor: false,
perf_map: Rc::default(),
perf_stack: vec![],
@@ -171,6 +175,23 @@ impl JITState {
unsafe { *(self.pc.offset(arg_idx + 1)) }
}
+ /// Return true if the current ISEQ could escape an environment.
+ ///
+ /// As of vm_push_frame(), EP is always equal to BP. However, after pushing
+ /// a frame, some ISEQ setups call vm_bind_update_env(), which redirects EP.
+ /// Also, some method calls escape the environment to the heap.
+ fn escapes_ep(&self) -> bool {
+ match unsafe { get_iseq_body_type(self.iseq) } {
+ // <main> frame is always associated to TOPLEVEL_BINDING.
+ ISEQ_TYPE_MAIN |
+ // Kernel#eval uses a heap EP when a Binding argument is not nil.
+ ISEQ_TYPE_EVAL => true,
+ // If this ISEQ has previously escaped EP, give up the optimization.
+ _ if iseq_escapes_ep(self.iseq) => true,
+ _ => false,
+ }
+ }
+
// Get the index of the next instruction
fn next_insn_idx(&self) -> u16 {
self.insn_idx + insn_len(self.get_opcode()) as u16
@@ -229,6 +250,33 @@ impl JITState {
}
}
+ pub fn assume_expected_cfunc(
+ &mut self,
+ asm: &mut Assembler,
+ ocb: &mut OutlinedCb,
+ class: VALUE,
+ method: ID,
+ cfunc: *mut c_void,
+ ) -> bool {
+ let cme = unsafe { rb_callable_method_entry(class, method) };
+
+ if cme.is_null() {
+ return false;
+ }
+
+ let def_type = unsafe { get_cme_def_type(cme) };
+ if def_type != VM_METHOD_TYPE_CFUNC {
+ return false;
+ }
+ if unsafe { get_mct_func(get_cme_def_body_cfunc(cme)) } != cfunc {
+ return false;
+ }
+
+ self.assume_method_lookup_stable(asm, ocb, cme);
+
+ true
+ }
+
pub fn assume_method_lookup_stable(&mut self, asm: &mut Assembler, ocb: &mut OutlinedCb, cme: CmePtr) -> Option<()> {
jit_ensure_block_entry_exit(self, asm, ocb)?;
self.method_lookup_assumptions.push(cme);
@@ -250,6 +298,19 @@ impl JITState {
true
}
+ /// Assume that base pointer is equal to environment pointer in the current ISEQ.
+ /// Return true if it's safe to assume so.
+ fn assume_no_ep_escape(&mut self, asm: &mut Assembler, ocb: &mut OutlinedCb) -> bool {
+ if jit_ensure_block_entry_exit(self, asm, ocb).is_none() {
+ return false; // out of space, give up
+ }
+ if self.escapes_ep() {
+ return false; // EP has been escaped in this ISEQ. disable the optimization to avoid an invalidation loop.
+ }
+ self.no_ep_escape = true;
+ true
+ }
+
fn get_cfp(&self) -> *mut rb_control_frame_struct {
unsafe { get_ec_cfp(self.ec) }
}
@@ -533,14 +594,36 @@ fn verify_ctx(jit: &JITState, ctx: &Context) {
unsafe { CStr::from_ptr(rb_obj_info(val)).to_str().unwrap() }
}
+ // Some types such as CString only assert the class field of the object
+ // when there has never been a singleton class created for objects of that class.
+ // Once there is a singleton class created they become their weaker
+ // `T*` variant, and we more objects should pass the verification.
+ fn relax_type_with_singleton_class_assumption(ty: Type) -> Type {
+ if let Type::CString | Type::CArray | Type::CHash = ty {
+ if has_singleton_class_of(ty.known_class().unwrap()) {
+ match ty {
+ Type::CString => return Type::TString,
+ Type::CArray => return Type::TArray,
+ Type::CHash => return Type::THash,
+ _ => (),
+ }
+ }
+ }
+
+ ty
+ }
+
// Only able to check types when at current insn
assert!(jit.at_current_insn());
let self_val = jit.peek_at_self();
let self_val_type = Type::from(self_val);
+ let learned_self_type = ctx.get_opnd_type(SelfOpnd);
+ let learned_self_type = relax_type_with_singleton_class_assumption(learned_self_type);
+
// Verify self operand type
- if self_val_type.diff(ctx.get_opnd_type(SelfOpnd)) == TypeDiff::Incompatible {
+ if self_val_type.diff(learned_self_type) == TypeDiff::Incompatible {
panic!(
"verify_ctx: ctx self type ({:?}) incompatible with actual value of self {}",
ctx.get_opnd_type(SelfOpnd),
@@ -553,6 +636,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) {
for i in 0..top_idx {
let learned_mapping = ctx.get_opnd_mapping(StackOpnd(i));
let learned_type = ctx.get_opnd_type(StackOpnd(i));
+ let learned_type = relax_type_with_singleton_class_assumption(learned_type);
let stack_val = jit.peek_at_stack(ctx, i as isize);
let val_type = Type::from(stack_val);
@@ -598,6 +682,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) {
let top_idx: usize = cmp::min(local_table_size as usize, MAX_TEMP_TYPES);
for i in 0..top_idx {
let learned_type = ctx.get_local_type(i);
+ let learned_type = relax_type_with_singleton_class_assumption(learned_type);
let local_val = jit.peek_at_local(i as i32);
let local_type = Type::from(local_val);
@@ -1384,7 +1469,7 @@ fn gen_putobject(
Some(KeepCompiling)
}
-/// Combine `putobject` and and `opt_ltlt` together if profitable, for example when
+/// Combine `putobject` and `opt_ltlt` together if profitable, for example when
/// left shifting an integer by a constant amount.
fn fuse_putobject_opt_ltlt(
jit: &mut JITState,
@@ -2203,16 +2288,22 @@ fn gen_get_lep(jit: &JITState, asm: &mut Assembler) -> Opnd {
fn gen_getlocal_generic(
jit: &mut JITState,
asm: &mut Assembler,
+ ocb: &mut OutlinedCb,
ep_offset: u32,
level: u32,
) -> Option<CodegenStatus> {
- // Load environment pointer EP (level 0) from CFP
- let ep_opnd = gen_get_ep(asm, level);
+ let local_opnd = if level == 0 && jit.assume_no_ep_escape(asm, ocb) {
+ // Load the local using SP register
+ asm.ctx.ep_opnd(-(ep_offset as i32))
+ } else {
+ // Load environment pointer EP (level 0) from CFP
+ let ep_opnd = gen_get_ep(asm, level);
- // Load the local from the block
- // val = *(vm_get_ep(GET_EP(), level) - idx);
- let offs = -(SIZEOF_VALUE_I32 * ep_offset as i32);
- let local_opnd = Opnd::mem(64, ep_opnd, offs);
+ // Load the local from the block
+ // val = *(vm_get_ep(GET_EP(), level) - idx);
+ let offs = -(SIZEOF_VALUE_I32 * ep_offset as i32);
+ Opnd::mem(64, ep_opnd, offs)
+ };
// Write the local at SP
let stack_top = if level == 0 {
@@ -2230,29 +2321,29 @@ fn gen_getlocal_generic(
fn gen_getlocal(
jit: &mut JITState,
asm: &mut Assembler,
- _ocb: &mut OutlinedCb,
+ ocb: &mut OutlinedCb,
) -> Option<CodegenStatus> {
let idx = jit.get_arg(0).as_u32();
let level = jit.get_arg(1).as_u32();
- gen_getlocal_generic(jit, asm, idx, level)
+ gen_getlocal_generic(jit, asm, ocb, idx, level)
}
fn gen_getlocal_wc0(
jit: &mut JITState,
asm: &mut Assembler,
- _ocb: &mut OutlinedCb,
+ ocb: &mut OutlinedCb,
) -> Option<CodegenStatus> {
let idx = jit.get_arg(0).as_u32();
- gen_getlocal_generic(jit, asm, idx, 0)
+ gen_getlocal_generic(jit, asm, ocb, idx, 0)
}
fn gen_getlocal_wc1(
jit: &mut JITState,
asm: &mut Assembler,
- _ocb: &mut OutlinedCb,
+ ocb: &mut OutlinedCb,
) -> Option<CodegenStatus> {
let idx = jit.get_arg(0).as_u32();
- gen_getlocal_generic(jit, asm, idx, 1)
+ gen_getlocal_generic(jit, asm, ocb, idx, 1)
}
fn gen_setlocal_generic(
@@ -2264,11 +2355,11 @@ fn gen_setlocal_generic(
) -> Option<CodegenStatus> {
let value_type = asm.ctx.get_opnd_type(StackOpnd(0));
- // Load environment pointer EP at level
- let ep_opnd = gen_get_ep(asm, level);
-
// Fallback because of write barrier
if asm.ctx.get_chain_depth() > 0 {
+ // Load environment pointer EP at level
+ let ep_opnd = gen_get_ep(asm, level);
+
// This function should not yield to the GC.
// void rb_vm_env_write(const VALUE *ep, int index, VALUE v)
let index = -(ep_offset as i64);
@@ -2286,16 +2377,27 @@ fn gen_setlocal_generic(
return Some(KeepCompiling);
}
- // Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers
- // only affect heap objects being written. If we know an immediate value is being written we
- // can skip this check.
- if !value_type.is_imm() {
- // flags & VM_ENV_FLAG_WB_REQUIRED
+ let (flags_opnd, local_opnd) = if level == 0 && jit.assume_no_ep_escape(asm, ocb) {
+ // Load flags and the local using SP register
+ let local_opnd = asm.ctx.ep_opnd(-(ep_offset as i32));
+ let flags_opnd = asm.ctx.ep_opnd(VM_ENV_DATA_INDEX_FLAGS as i32);
+ (flags_opnd, local_opnd)
+ } else {
+ // Load flags and the local for the level
+ let ep_opnd = gen_get_ep(asm, level);
let flags_opnd = Opnd::mem(
64,
ep_opnd,
SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_FLAGS as i32,
);
+ (flags_opnd, Opnd::mem(64, ep_opnd, -SIZEOF_VALUE_I32 * ep_offset as i32))
+ };
+
+ // Write barriers may be required when VM_ENV_FLAG_WB_REQUIRED is set, however write barriers
+ // only affect heap objects being written. If we know an immediate value is being written we
+ // can skip this check.
+ if !value_type.is_imm() {
+ // flags & VM_ENV_FLAG_WB_REQUIRED
asm.test(flags_opnd, VM_ENV_FLAG_WB_REQUIRED.into());
// if (flags & VM_ENV_FLAG_WB_REQUIRED) != 0
@@ -2319,8 +2421,7 @@ fn gen_setlocal_generic(
let stack_top = asm.stack_pop(1);
// Write the value at the environment pointer
- let offs = -(SIZEOF_VALUE_I32 * ep_offset as i32);
- asm.mov(Opnd::mem(64, ep_opnd, offs), stack_top);
+ asm.mov(local_opnd, stack_top);
Some(KeepCompiling)
}
@@ -4134,11 +4235,50 @@ fn gen_opt_newarray_send(
gen_opt_newarray_max(jit, asm, _ocb)
} else if method == ID!(hash) {
gen_opt_newarray_hash(jit, asm, _ocb)
+ } else if method == ID!(pack) {
+ gen_opt_newarray_pack(jit, asm, _ocb)
} else {
None
}
}
+fn gen_opt_newarray_pack(
+ jit: &mut JITState,
+ asm: &mut Assembler,
+ _ocb: &mut OutlinedCb,
+) -> Option<CodegenStatus> {
+ // num == 4 ( for this code )
+ let num = jit.get_arg(0).as_u32();
+
+ // Save the PC and SP because we may call #pack
+ jit_prepare_non_leaf_call(jit, asm);
+
+ extern "C" {
+ fn rb_vm_opt_newarray_pack(ec: EcPtr, num: u32, elts: *const VALUE, fmt: VALUE) -> VALUE;
+ }
+
+ let values_opnd = asm.ctx.sp_opnd(-(num as i32));
+ let values_ptr = asm.lea(values_opnd);
+
+ let fmt_string = asm.ctx.sp_opnd(-1);
+
+ let val_opnd = asm.ccall(
+ rb_vm_opt_newarray_pack as *const u8,
+ vec![
+ EC,
+ (num - 1).into(),
+ values_ptr,
+ fmt_string
+ ],
+ );
+
+ asm.stack_pop(num.as_usize());
+ let stack_ret = asm.stack_push(Type::CString);
+ asm.mov(stack_ret, val_opnd);
+
+ Some(KeepCompiling)
+}
+
fn gen_opt_newarray_hash(
jit: &mut JITState,
asm: &mut Assembler,
@@ -5471,7 +5611,7 @@ fn jit_rb_str_uplus(
let recv_opnd = asm.stack_pop(1);
let recv_opnd = asm.load(recv_opnd);
let flags_opnd = asm.load(Opnd::mem(64, recv_opnd, RUBY_OFFSET_RBASIC_FLAGS));
- asm.test(flags_opnd, Opnd::Imm(RUBY_FL_FREEZE as i64));
+ asm.test(flags_opnd, Opnd::Imm(RUBY_FL_FREEZE as i64 | RSTRING_CHILLED as i64));
let ret_label = asm.new_label("stack_ret");
@@ -5879,8 +6019,10 @@ fn jit_rb_ary_push(
) -> bool {
asm_comment!(asm, "Array#<<");
- // rb_ary_push allocates memory for buffer extension
- jit_prepare_call_with_gc(jit, asm);
+ // rb_ary_push allocates memory for buffer extension and can raise FrozenError
+ // Not using a lazy frame here since the interpreter also has a truncated
+ // stack trace from opt_ltlt.
+ jit_prepare_non_leaf_call(jit, asm);
let item_opnd = asm.stack_opnd(0);
let ary_opnd = asm.stack_opnd(1);
@@ -6058,6 +6200,64 @@ fn gen_block_given(
asm.mov(out_opnd, block_given);
}
+// Codegen for rb_class_superclass()
+fn jit_rb_class_superclass(
+ jit: &mut JITState,
+ asm: &mut Assembler,
+ _ocb: &mut OutlinedCb,
+ _ci: *const rb_callinfo,
+ cme: *const rb_callable_method_entry_t,
+ _block: Option<crate::codegen::BlockHandler>,
+ _argc: i32,
+ _known_recv_class: Option<VALUE>,
+) -> bool {
+ extern "C" {
+ fn rb_class_superclass(klass: VALUE) -> VALUE;
+ }
+
+ if !jit_prepare_lazy_frame_call(jit, asm, cme, StackOpnd(0)) {
+ return false;
+ }
+
+ asm_comment!(asm, "Class#superclass");
+ let recv_opnd = asm.stack_opnd(0);
+ let ret = asm.ccall(rb_class_superclass as *const u8, vec![recv_opnd]);
+
+ asm.stack_pop(1);
+ let ret_opnd = asm.stack_push(Type::Unknown);
+ asm.mov(ret_opnd, ret);
+
+ true
+}
+
+fn jit_rb_case_equal(
+ jit: &mut JITState,
+ asm: &mut Assembler,
+ ocb: &mut OutlinedCb,
+ _ci: *const rb_callinfo,
+ _cme: *const rb_callable_method_entry_t,
+ _block: Option<BlockHandler>,
+ _argc: i32,
+ known_recv_class: Option<VALUE>,
+) -> bool {
+ if !jit.assume_expected_cfunc( asm, ocb, known_recv_class.unwrap(), ID!(eq), rb_obj_equal as _) {
+ return false;
+ }
+
+ asm_comment!(asm, "case_equal: {}#===", get_class_name(known_recv_class));
+
+ // Compare the arguments
+ let arg1 = asm.stack_pop(1);
+ let arg0 = asm.stack_pop(1);
+ asm.cmp(arg0, arg1);
+ let ret_opnd = asm.csel_e(Qtrue.into(), Qfalse.into());
+
+ let stack_ret = asm.stack_push(Type::UnknownImm);
+ asm.mov(stack_ret, ret_opnd);
+
+ true
+}
+
fn jit_thread_s_current(
_jit: &mut JITState,
asm: &mut Assembler,
@@ -6753,20 +6953,20 @@ fn push_splat_args(required_args: u32, asm: &mut Assembler) {
asm.cmp(array_len_opnd, required_args.into());
asm.jne(Target::side_exit(Counter::guard_send_splatarray_length_not_equal));
- asm_comment!(asm, "Check last argument is not ruby2keyword hash");
-
- // Need to repeat this here to deal with register allocation
- let array_reg = asm.load(asm.stack_opnd(0));
-
- let ary_opnd = get_array_ptr(asm, array_reg);
-
- let last_array_value = asm.load(Opnd::mem(64, ary_opnd, (required_args as i32 - 1) * (SIZEOF_VALUE as i32)));
+ // Check last element of array if present
+ if required_args > 0 {
+ asm_comment!(asm, "Check last argument is not ruby2keyword hash");
- guard_object_is_not_ruby2_keyword_hash(
- asm,
- last_array_value,
- Counter::guard_send_splatarray_last_ruby2_keywords,
- );
+ // Need to repeat this here to deal with register allocation
+ let array_reg = asm.load(asm.stack_opnd(0));
+ let ary_opnd = get_array_ptr(asm, array_reg);
+ let last_array_value = asm.load(Opnd::mem(64, ary_opnd, (required_args as i32 - 1) * (SIZEOF_VALUE as i32)));
+ guard_object_is_not_ruby2_keyword_hash(
+ asm,
+ last_array_value,
+ Counter::guard_send_splatarray_last_ruby2_keywords,
+ );
+ }
asm_comment!(asm, "Push arguments from array");
let array_opnd = asm.stack_pop(1);
@@ -8148,6 +8348,13 @@ fn gen_struct_aref(
}
}
+ if c_method_tracing_currently_enabled(jit) {
+ // Struct accesses need fire c_call and c_return events, which we can't support
+ // See :attr-tracing:
+ gen_counter_incr(asm, Counter::send_cfunc_tracing);
+ return None;
+ }
+
// This is a .send call and we need to adjust the stack
if flags & VM_CALL_OPT_SEND != 0 {
handle_opt_send_shift_stack(asm, argc);
@@ -8192,6 +8399,13 @@ fn gen_struct_aset(
return None;
}
+ if c_method_tracing_currently_enabled(jit) {
+ // Struct accesses need fire c_call and c_return events, which we can't support
+ // See :attr-tracing:
+ gen_counter_incr(asm, Counter::send_cfunc_tracing);
+ return None;
+ }
+
// This is a .send call and we need to adjust the stack
if flags & VM_CALL_OPT_SEND != 0 {
handle_opt_send_shift_stack(asm, argc);
@@ -8458,10 +8672,8 @@ fn gen_send_general(
// Handling the C method tracing events for attr_accessor
// methods is easier than regular C methods as we know the
// "method" we are calling into never enables those tracing
- // events. Once global invalidation runs, the code for the
- // attr_accessor is invalidated and we exit at the closest
- // instruction boundary which is always outside of the body of
- // the attr_accessor code.
+ // events. We are never inside the code that needs to be
+ // invalidated when invalidation happens.
gen_counter_incr(asm, Counter::send_cfunc_tracing);
return None;
}
@@ -8721,11 +8933,16 @@ fn gen_send_general(
}
}
+/// Get class name from a class pointer.
+fn get_class_name(class: Option<VALUE>) -> String {
+ class.and_then(|class| unsafe {
+ cstr_to_rust_string(rb_class2name(class))
+ }).unwrap_or_else(|| "Unknown".to_string())
+}
+
/// Assemble "{class_name}#{method_name}" from a class pointer and a method ID
fn get_method_name(class: Option<VALUE>, mid: u64) -> String {
- let class_name = class.and_then(|class| unsafe {
- cstr_to_rust_string(rb_class2name(class))
- }).unwrap_or_else(|| "Unknown".to_string());
+ let class_name = get_class_name(class);
let method_name = if mid != 0 {
unsafe { cstr_to_rust_string(rb_id2name(mid)) }
} else {
@@ -10060,6 +10277,10 @@ pub fn yjit_reg_method_codegen_fns() {
yjit_reg_method(rb_cString, "<<", jit_rb_str_concat);
yjit_reg_method(rb_cString, "+@", jit_rb_str_uplus);
+ yjit_reg_method(rb_cNilClass, "===", jit_rb_case_equal);
+ yjit_reg_method(rb_cTrueClass, "===", jit_rb_case_equal);
+ yjit_reg_method(rb_cFalseClass, "===", jit_rb_case_equal);
+
yjit_reg_method(rb_cArray, "empty?", jit_rb_ary_empty_p);
yjit_reg_method(rb_cArray, "length", jit_rb_ary_length);
yjit_reg_method(rb_cArray, "size", jit_rb_ary_length);
@@ -10070,6 +10291,8 @@ pub fn yjit_reg_method_codegen_fns() {
yjit_reg_method(rb_mKernel, "respond_to?", jit_obj_respond_to);
yjit_reg_method(rb_mKernel, "block_given?", jit_rb_f_block_given_p);
+ yjit_reg_method(rb_cClass, "superclass", jit_rb_class_superclass);
+
yjit_reg_method(rb_singleton_class(rb_cThread), "current", jit_thread_s_current);
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-01 16:50:08 +0000