diff options
| author | Jeff Zhang <jeff.j.zhang@shopify.com> | 2026-01-29 13:00:13 -0500 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-01-29 18:00:13 +0000 |
| commit | fbff0c936cc8b1a1c5f6fe51ea6e892721c200f1 (patch) | |
| tree | bbaffe08e6313248081c0cabe19a0379fb0a5cff | |
| parent | 5299276e421c3f11c944069fe5c655628634b9e1 (diff) | |
ZJIT: Handle `nil` case for `getblockparamproxy` (#15986)
Resolves https://github.com/Shopify/ruby/issues/772
Adds profiling for the `getblockparamproxy` YARV instruction and handles the `nil` block case by pushing `nil` instead of the block proxy object, improves `ratio_in_zjit` a tiny bit (0.1%)
Profiling data for `getblockparamproxy` on Lobsters
```
Top-6 getblockparamproxy handler (100.0% of total 3,353,291):
polymorphic: 2,337,372 (69.7%)
nil: 552,629 (16.5%)
iseq: 259,636 ( 7.7%)
no_profiles: 156,734 ( 4.7%)
proc: 40,223 ( 1.2%)
megamorphic: 6,697 ( 0.2%)
```
Lobsters benchmark stats:
<details>
<summary>Stats before (master):</summary>
<p>
```
❯ ./run_benchmarks.rb --chruby 'ruby-zjit --zjit-stats' lobsters
***ZJIT: Printing ZJIT statistics on exit***
...
Top-20 side exit reasons (100.0% of total 15,338,024):
guard_type_failure: 6,889,050 (44.9%)
guard_shape_failure: 6,848,898 (44.7%)
block_param_proxy_not_iseq_or_ifunc: 1,008,525 ( 6.6%)
unhandled_hir_insn: 236,977 ( 1.5%)
compile_error: 191,763 ( 1.3%)
fixnum_mult_overflow: 50,739 ( 0.3%)
block_param_proxy_modified: 28,119 ( 0.2%)
patchpoint_stable_constant_names: 18,229 ( 0.1%)
unhandled_newarray_send_pack: 14,481 ( 0.1%)
unhandled_block_arg: 13,782 ( 0.1%)
fixnum_lshift_overflow: 10,085 ( 0.1%)
patchpoint_no_ep_escape: 7,815 ( 0.1%)
unhandled_yarv_insn: 7,540 ( 0.0%)
expandarray_failure: 4,533 ( 0.0%)
guard_super_method_entry: 4,475 ( 0.0%)
patchpoint_method_redefined: 1,207 ( 0.0%)
patchpoint_no_singleton_class: 1,130 ( 0.0%)
obj_to_string_fallback: 412 ( 0.0%)
guard_less_failure: 163 ( 0.0%)
interrupt: 82 ( 0.0%)
...
ratio_in_zjit: 82.1%
```
</p>
</details>
<details>
<summary>Stats after:</summary>
<p>
```
❯ ./run_benchmarks.rb --chruby 'ruby-zjit --zjit-stats' lobsters
***ZJIT: Printing ZJIT statistics on exit***
...
Top-20 side exit reasons (100.0% of total 15,061,422):
guard_type_failure: 6,892,934 (45.8%)
guard_shape_failure: 6,850,512 (45.5%)
block_param_proxy_not_iseq_or_ifunc: 549,823 ( 3.7%)
unhandled_hir_insn: 236,979 ( 1.6%)
compile_error: 191,782 ( 1.3%)
unhandled_yarv_insn: 128,695 ( 0.9%)
block_param_proxy_not_nil: 68,623 ( 0.5%)
fixnum_mult_overflow: 50,739 ( 0.3%)
patchpoint_stable_constant_names: 18,568 ( 0.1%)
unhandled_newarray_send_pack: 14,481 ( 0.1%)
block_param_proxy_modified: 13,819 ( 0.1%)
unhandled_block_arg: 13,798 ( 0.1%)
fixnum_lshift_overflow: 10,085 ( 0.1%)
patchpoint_no_ep_escape: 7,815 ( 0.1%)
expandarray_failure: 4,533 ( 0.0%)
guard_super_method_entry: 4,475 ( 0.0%)
patchpoint_method_redefined: 1,207 ( 0.0%)
obj_to_string_fallback: 1,140 ( 0.0%)
patchpoint_no_singleton_class: 1,130 ( 0.0%)
guard_less_failure: 163 ( 0.0%)
...
ratio_in_zjit: 82.2%
```
</p>
</details>
| -rw-r--r-- | insns.def | 1 | ||||
| -rw-r--r-- | vm_insnhelper.c | 2 | ||||
| -rw-r--r-- | yjit/src/cruby_bindings.inc.rs | 65 | ||||
| -rw-r--r-- | zjit.c | 1 | ||||
| -rw-r--r-- | zjit.rb | 1 | ||||
| -rw-r--r-- | zjit/bindgen/src/main.rs | 1 | ||||
| -rw-r--r-- | zjit/src/codegen.rs | 50 | ||||
| -rw-r--r-- | zjit/src/cruby.rs | 2 | ||||
| -rw-r--r-- | zjit/src/cruby_bindings.inc.rs | 66 | ||||
| -rw-r--r-- | zjit/src/hir.rs | 121 | ||||
| -rw-r--r-- | zjit/src/hir/opt_tests.rs | 93 | ||||
| -rw-r--r-- | zjit/src/hir/tests.rs | 40 | ||||
| -rw-r--r-- | zjit/src/profile.rs | 17 | ||||
| -rw-r--r-- | zjit/src/stats.rs | 11 |
14 files changed, 344 insertions, 127 deletions
@@ -145,6 +145,7 @@ getblockparamproxy (lindex_t idx, rb_num_t level) () (VALUE val) +// attr bool zjit_profile = true; { const VALUE *ep = vm_get_ep(GET_EP(), level); VM_ASSERT(VM_ENV_LOCAL_P(ep)); diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 9cb163e97f..a27bf5f49b 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -6050,7 +6050,7 @@ vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqv // * If it's VM_BLOCK_HANDLER_NONE, return nil // * If it's an ISEQ or an IFUNC, fetch it from its rb_captured_block // * If it's a PROC or SYMBOL, return it as is -static VALUE +VALUE rb_vm_untag_block_handler(VALUE block_handler) { if (VM_BLOCK_HANDLER_NONE == block_handler) return Qnil; diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 56994388a3..9216802a3c 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -917,38 +917,39 @@ pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 214; pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 215; pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 216; pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 217; -pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 218; -pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 219; -pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 220; -pub const YARVINSN_zjit_send: ruby_vminsn_type = 221; -pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 222; -pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 223; -pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 224; -pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 225; -pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 226; -pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 227; -pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 228; -pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 229; -pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 230; -pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 231; -pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 232; -pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 233; -pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 234; -pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 235; -pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 236; -pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 237; -pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 238; -pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 239; -pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 240; -pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 241; -pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 242; -pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 243; -pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 244; -pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 245; -pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 246; -pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 247; -pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 248; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 249; +pub const YARVINSN_zjit_getblockparamproxy: ruby_vminsn_type = 218; +pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 219; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 220; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 221; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 222; +pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 223; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 224; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 225; +pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 248; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 249; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 250; pub type ruby_vminsn_type = u32; pub type rb_iseq_callback = ::std::option::Option< unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void), @@ -300,6 +300,7 @@ rb_zjit_class_has_default_allocator(VALUE klass) } +VALUE rb_vm_untag_block_handler(VALUE block_handler); VALUE rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp); void @@ -191,6 +191,7 @@ class << RubyVM::ZJIT print_counters_with_prefix(prefix: 'getivar_fallback_', prompt: 'getivar fallback reasons', buf:, stats:, limit: 5) print_counters_with_prefix(prefix: 'definedivar_fallback_', prompt: 'definedivar fallback reasons', buf:, stats:, limit: 5) print_counters_with_prefix(prefix: 'invokeblock_handler_', prompt: 'invokeblock handler', buf:, stats:, limit: 10) + print_counters_with_prefix(prefix: 'getblockparamproxy_handler_', prompt: 'getblockparamproxy handler', buf:, stats:, limit: 10) # Show most popular unsupported call features. Because each call can # use multiple complex features, a decrease in this number does not diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 794293d1d3..d71e75c444 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -409,6 +409,7 @@ fn main() { .allowlist_function("rb_str_neq_internal") .allowlist_function("rb_yarv_ary_entry_internal") .allowlist_function("rb_vm_get_untagged_block_handler") + .allowlist_function("rb_vm_untag_block_handler") .allowlist_function("rb_FL_TEST") .allowlist_function("rb_FL_TEST_RAW") .allowlist_function("rb_RB_TYPE_P") diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 5d6060dd49..9276d0af6b 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -529,7 +529,8 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::GuardType { val, guard_type, state } => gen_guard_type(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), Insn::GuardTypeNot { val, guard_type, state } => gen_guard_type_not(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), &Insn::GuardBitEquals { val, expected, reason, state } => gen_guard_bit_equals(jit, asm, opnd!(val), expected, reason, &function.frame_state(state)), - &Insn::GuardBlockParamProxy { level, state } => no_output!(gen_guard_block_param_proxy(jit, asm, level, &function.frame_state(state))), + &Insn::GuardAnyBitSet { val, mask, reason, state } => gen_guard_any_bit_set(jit, asm, opnd!(val), mask, reason, &function.frame_state(state)), + &Insn::GuardNoBitsSet { val, mask, reason, state } => gen_guard_no_bits_set(jit, asm, opnd!(val), mask, reason, &function.frame_state(state)), Insn::GuardNotFrozen { recv, state } => gen_guard_not_frozen(jit, asm, opnd!(recv), &function.frame_state(*state)), Insn::GuardNotShared { recv, state } => gen_guard_not_shared(jit, asm, opnd!(recv), &function.frame_state(*state)), &Insn::GuardLess { left, right, state } => gen_guard_less(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), @@ -580,6 +581,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::GuardShape { val, shape, state } => gen_guard_shape(jit, asm, opnd!(val), shape, &function.frame_state(state)), Insn::LoadPC => gen_load_pc(asm), Insn::LoadEC => gen_load_ec(), + &Insn::GetEP { level } => gen_get_ep(asm, level), Insn::GetLEP => gen_get_lep(jit, asm), Insn::LoadSelf => gen_load_self(), &Insn::LoadField { recv, id, offset, return_type } => gen_load_field(asm, opnd!(recv), id, offset, return_type), @@ -786,26 +788,6 @@ fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, ep_offset: u32, le asm.load(Opnd::mem(VALUE_BITS, ep, offset)) } -fn gen_guard_block_param_proxy(jit: &JITState, asm: &mut Assembler, level: u32, state: &FrameState) { - // Bail out if the `&block` local variable has been modified - let ep = gen_get_ep(asm, level); - let flags = Opnd::mem(64, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); - asm.test(flags, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); - asm.jnz(side_exit(jit, state, SideExitReason::BlockParamProxyModified)); - - // This handles two cases which are nearly identical - // Block handler is a tagged pointer. Look at the tag. - // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 - // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 - // So to check for either of those cases we can use: val & 0x1 == 0x1 - const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); - - // Bail ouf if the block handler is neither ISEQ nor ifunc - let block_handler = asm.load(Opnd::mem(64, ep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); - asm.test(block_handler, 0x1.into()); - asm.jz(side_exit(jit, state, SideExitReason::BlockParamProxyNotIseqOrIfunc)); -} - fn gen_guard_not_frozen(jit: &JITState, asm: &mut Assembler, recv: Opnd, state: &FrameState) -> Opnd { let recv = asm.load(recv); // It's a heap object, so check the frozen flag @@ -2338,6 +2320,32 @@ fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val } +fn mask_to_opnd(mask: crate::hir::Const) -> Option<Opnd> { + match mask { + crate::hir::Const::CUInt8(v) => Some(Opnd::UImm(v as u64)), + crate::hir::Const::CUInt16(v) => Some(Opnd::UImm(v as u64)), + crate::hir::Const::CUInt32(v) => Some(Opnd::UImm(v as u64)), + crate::hir::Const::CUInt64(v) => Some(Opnd::UImm(v)), + _ => None + } +} + +/// Compile a bitmask check with a side exit if none of the masked bits are not set +fn gen_guard_any_bit_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { + let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_any_bit_set: unexpected hir::Const {mask:?}")); + asm.test(val, mask_opnd); + asm.jz(side_exit(jit, state, reason)); + val +} + +/// Compile a bitmask check with a side exit if any of the masked bits are set +fn gen_guard_no_bits_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { + let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_no_bits_set: unexpected hir::Const {mask:?}")); + asm.test(val, mask_opnd); + asm.jnz(side_exit(jit, state, reason)); + val +} + /// Generate code that records unoptimized C functions if --zjit-stats is enabled fn gen_incr_counter_ptr(asm: &mut Assembler, counter_ptr: *mut u64) { if get_option!(stats) { diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 51faaab9c2..94b2a443c8 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -1396,6 +1396,8 @@ pub(crate) mod ids { name: self_ content: b"self" name: rb_ivar_get_at_no_ractor_check name: _shape_id + name: _env_data_index_flags + name: _env_data_index_specval } /// Get an CRuby `ID` to an interned string, e.g. a particular method name. diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 15533180da..969c5a4c69 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1706,38 +1706,39 @@ pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 214; pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 215; pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 216; pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 217; -pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 218; -pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 219; -pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 220; -pub const YARVINSN_zjit_send: ruby_vminsn_type = 221; -pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 222; -pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 223; -pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 224; -pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 225; -pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 226; -pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 227; -pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 228; -pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 229; -pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 230; -pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 231; -pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 232; -pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 233; -pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 234; -pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 235; -pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 236; -pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 237; -pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 238; -pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 239; -pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 240; -pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 241; -pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 242; -pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 243; -pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 244; -pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 245; -pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 246; -pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 247; -pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 248; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 249; +pub const YARVINSN_zjit_getblockparamproxy: ruby_vminsn_type = 218; +pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 219; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 220; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 221; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 222; +pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 223; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 224; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 225; +pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 248; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 249; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 250; pub type ruby_vminsn_type = u32; pub type rb_iseq_callback = ::std::option::Option< unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void), @@ -2081,6 +2082,7 @@ unsafe extern "C" { pub fn rb_zjit_class_initialized_p(klass: VALUE) -> bool; pub fn rb_zjit_class_get_alloc_func(klass: VALUE) -> rb_alloc_func_t; pub fn rb_zjit_class_has_default_allocator(klass: VALUE) -> bool; + pub fn rb_vm_untag_block_handler(block_handler: VALUE) -> VALUE; pub fn rb_vm_get_untagged_block_handler(reg_cfp: *mut rb_control_frame_t) -> VALUE; pub fn rb_zjit_writebarrier_check_immediate(recv: VALUE, val: VALUE); pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index b523d8430f..32519a5b97 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -506,6 +506,7 @@ pub enum SideExitReason { Interrupt, BlockParamProxyModified, BlockParamProxyNotIseqOrIfunc, + BlockParamProxyNotNil, BlockParamWbRequired, StackOverflow, FixnumModByZero, @@ -855,6 +856,9 @@ pub enum Insn { /// Set a class variable `id` to `val` SetClassVar { id: ID, val: InsnId, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, + /// Get the EP at the given level from the current CFP. + GetEP { level: u32 }, + /// Get the EP of the ISeq of the containing method, or "local level", skipping over block-level EPs. /// Equivalent of GET_LEP() macro. GetLEP, @@ -1018,11 +1022,12 @@ pub enum Insn { GuardTypeNot { val: InsnId, guard_type: Type, state: InsnId }, /// Side-exit if val is not the expected Const. GuardBitEquals { val: InsnId, expected: Const, reason: SideExitReason, state: InsnId }, + /// Side-exit if (val & mask) == 0 + GuardAnyBitSet { val: InsnId, mask: Const, reason: SideExitReason, state: InsnId }, + /// Side-exit if (val & mask) != 0 + GuardNoBitsSet { val: InsnId, mask: Const, reason: SideExitReason, state: InsnId }, /// Side-exit if val doesn't have the expected shape. GuardShape { val: InsnId, shape: ShapeId, state: InsnId }, - /// Side-exit if the block param has been modified or the block handler for the frame - /// is neither ISEQ nor ifunc, which makes it incompatible with rb_block_param_proxy. - GuardBlockParamProxy { level: u32, state: InsnId }, /// Side-exit if val is frozen. Does *not* check if the val is an immediate; assumes that it is /// a heap object. GuardNotFrozen { recv: InsnId, state: InsnId }, @@ -1066,7 +1071,7 @@ impl Insn { | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetGlobal { .. } | Insn::SetLocal { .. } | Insn::Throw { .. } | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } - | Insn::CheckInterrupts { .. } | Insn::GuardBlockParamProxy { .. } | Insn::GuardSuperMethodEntry { .. } + | Insn::CheckInterrupts { .. } | Insn::GuardSuperMethodEntry { .. } | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } | Insn::ArrayAset { .. } => false, _ => true, @@ -1154,6 +1159,7 @@ impl Insn { Insn::DefinedIvar { .. } => effects::Any, Insn::LoadPC { .. } => Effect::read_write(abstract_heaps::PC, abstract_heaps::Empty), Insn::LoadEC { .. } => effects::Empty, + Insn::GetEP { .. } => effects::Empty, Insn::GetLEP { .. } => effects::Empty, Insn::LoadSelf { .. } => Effect::read_write(abstract_heaps::Frame, abstract_heaps::Empty), Insn::LoadField { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), @@ -1220,8 +1226,9 @@ impl Insn { Insn::GuardType { .. } => effects::Any, Insn::GuardTypeNot { .. } => effects::Any, Insn::GuardBitEquals { .. } => effects::Any, + Insn::GuardAnyBitSet { .. } => effects::Any, + Insn::GuardNoBitsSet { .. } => effects::Any, Insn::GuardShape { .. } => effects::Any, - Insn::GuardBlockParamProxy { .. } => effects::Any, Insn::GuardNotFrozen { .. } => effects::Any, Insn::GuardNotShared { .. } => effects::Any, Insn::GuardGreaterEq { .. } => effects::Any, @@ -1541,8 +1548,9 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::RefineType { val, new_type, .. } => { write!(f, "RefineType {val}, {}", new_type.print(self.ptr_map)) }, Insn::GuardTypeNot { val, guard_type, .. } => { write!(f, "GuardTypeNot {val}, {}", guard_type.print(self.ptr_map)) }, Insn::GuardBitEquals { val, expected, .. } => { write!(f, "GuardBitEquals {val}, {}", expected.print(self.ptr_map)) }, + Insn::GuardAnyBitSet { val, mask, .. } => { write!(f, "GuardBitSet {val}, {}", mask.print(self.ptr_map)) }, + Insn::GuardNoBitsSet { val, mask, .. } => { write!(f, "GuardBitNotSet {val}, {}", mask.print(self.ptr_map)) }, &Insn::GuardShape { val, shape, .. } => { write!(f, "GuardShape {val}, {:p}", self.ptr_map.map_shape(shape)) }, - Insn::GuardBlockParamProxy { level, .. } => write!(f, "GuardBlockParamProxy l{level}"), Insn::GuardNotFrozen { recv, .. } => write!(f, "GuardNotFrozen {recv}"), Insn::GuardNotShared { recv, .. } => write!(f, "GuardNotShared {recv}"), Insn::GuardLess { left, right, .. } => write!(f, "GuardLess {left}, {right}"), @@ -1604,6 +1612,7 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::GetIvar { self_val, id, .. } => write!(f, "GetIvar {self_val}, :{}", id.contents_lossy()), Insn::LoadPC => write!(f, "LoadPC"), Insn::LoadEC => write!(f, "LoadEC"), + &Insn::GetEP { level } => write!(f, "GetEP {level}"), Insn::GetLEP => write!(f, "GetLEP"), Insn::LoadSelf => write!(f, "LoadSelf"), &Insn::LoadField { recv, id, offset, return_type: _ } => write!(f, "LoadField {recv}, :{}@{:p}", id.contents_lossy(), self.ptr_map.map_offset(offset)), @@ -2187,6 +2196,7 @@ impl Function { | EntryPoint {..} | LoadPC | LoadEC + | GetEP {..} | GetLEP | LoadSelf | IncrCounterPtr {..} @@ -2228,8 +2238,9 @@ impl Function { &GuardType { val, guard_type, state } => GuardType { val: find!(val), guard_type, state }, &GuardTypeNot { val, guard_type, state } => GuardTypeNot { val: find!(val), guard_type, state }, &GuardBitEquals { val, expected, reason, state } => GuardBitEquals { val: find!(val), expected, reason, state }, + &GuardAnyBitSet { val, mask, reason, state } => GuardAnyBitSet { val: find!(val), mask, reason, state }, + &GuardNoBitsSet { val, mask, reason, state } => GuardNoBitsSet { val: find!(val), mask, reason, state }, &GuardShape { val, shape, state } => GuardShape { val: find!(val), shape, state }, - &GuardBlockParamProxy { level, state } => GuardBlockParamProxy { level, state: find!(state) }, &GuardNotFrozen { recv, state } => GuardNotFrozen { recv: find!(recv), state }, &GuardNotShared { recv, state } => GuardNotShared { recv: find!(recv), state }, &GuardGreaterEq { left, right, state } => GuardGreaterEq { left: find!(left), right: find!(right), state }, @@ -2429,7 +2440,7 @@ impl Function { | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetLocal { .. } | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } - | Insn::CheckInterrupts { .. } | Insn::GuardBlockParamProxy { .. } | Insn::GuardSuperMethodEntry { .. } + | Insn::CheckInterrupts { .. } | Insn::GuardSuperMethodEntry { .. } | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } | Insn::ArrayAset { .. } => panic!("Cannot infer type of instruction with no output: {}. See Insn::has_output().", self.insns[insn.0]), Insn::Const { val: Const::Value(val) } => Type::from_value(*val), @@ -2488,6 +2499,8 @@ impl Function { Insn::RefineType { val, new_type, .. } => self.type_of(*val).intersection(*new_type), Insn::GuardTypeNot { .. } => types::BasicObject, Insn::GuardBitEquals { val, expected, .. } => self.type_of(*val).intersection(Type::from_const(*expected)), + Insn::GuardAnyBitSet { val, .. } => self.type_of(*val), + Insn::GuardNoBitsSet { val, .. } => self.type_of(*val), Insn::GuardShape { val, .. } => self.type_of(*val), Insn::GuardNotFrozen { recv, .. } | Insn::GuardNotShared { recv, .. } => self.type_of(*recv), Insn::GuardLess { left, .. } => self.type_of(*left), @@ -2532,6 +2545,7 @@ impl Function { Insn::GetIvar { .. } => types::BasicObject, Insn::LoadPC => types::CPtr, Insn::LoadEC => types::CPtr, + Insn::GetEP { .. } => types::CPtr, Insn::GetLEP => types::CPtr, Insn::LoadSelf => types::BasicObject, &Insn::LoadField { return_type, .. } => return_type, @@ -2659,7 +2673,9 @@ impl Function { Insn::GuardType { val, .. } | Insn::GuardTypeNot { val, .. } | Insn::GuardShape { val, .. } - | Insn::GuardBitEquals { val, .. } => self.chase_insn(val), + | Insn::GuardBitEquals { val, .. } + | Insn::GuardAnyBitSet { val, .. } + | Insn::GuardNoBitsSet { val, .. } => self.chase_insn(val), | Insn::RefineType { val, .. } => self.chase_insn(val), _ => id, } @@ -4517,6 +4533,7 @@ impl Function { | &Insn::EntryPoint { .. } | &Insn::LoadPC | &Insn::LoadEC + | &Insn::GetEP { .. } | &Insn::GetLEP | &Insn::LoadSelf | &Insn::GetLocal { .. } @@ -4605,6 +4622,8 @@ impl Function { | &Insn::GuardType { val, state, .. } | &Insn::GuardTypeNot { val, state, .. } | &Insn::GuardBitEquals { val, state, .. } + | &Insn::GuardAnyBitSet { val, state, .. } + | &Insn::GuardNoBitsSet { val, state, .. } | &Insn::GuardShape { val, state, .. } | &Insn::GuardNotFrozen { recv: val, state } | &Insn::GuardNotShared { recv: val, state } @@ -4755,7 +4774,6 @@ impl Function { worklist.push_back(recv); worklist.push_back(val); } - &Insn::GuardBlockParamProxy { state, .. } | &Insn::GetGlobal { state, .. } | &Insn::GetSpecialSymbol { state, .. } | &Insn::GetSpecialNumber { state, .. } | @@ -5301,12 +5319,12 @@ impl Function { | Insn::GetGlobal { .. } | Insn::LoadPC | Insn::LoadEC + | Insn::GetEP { .. } | Insn::GetLEP | Insn::LoadSelf | Insn::Snapshot { .. } | Insn::Jump { .. } | Insn::EntryPoint { .. } - | Insn::GuardBlockParamProxy { .. } | Insn::GuardSuperMethodEntry { .. } | Insn::GetBlockHandler { .. } | Insn::PatchPoint { .. } @@ -5530,6 +5548,18 @@ impl Function { Const::CPtr(_) => self.assert_subtype(insn_id, val, types::CPtr), } } + Insn::GuardAnyBitSet { val, mask, .. } + | Insn::GuardNoBitsSet { val, mask, .. } => { + match mask { + Const::CUInt8(_) | Const::CUInt16(_) | Const::CUInt32(_) | Const::CUInt64(_) + if self.is_a(val, types::CInt) || self.is_a(val, types::RubyValue) => { + Ok(()) + } + _ => { + Err(ValidationError::MiscValidationError(insn_id, "GuardAnyBitSet/GuardNoBitsSet can only compare RubyValue/CUInt or CInt/CUInt".to_string())) + } + } + } Insn::GuardLess { left, right, .. } | Insn::GuardGreaterEq { left, right, .. } => { self.assert_subtype(insn_id, left, types::CInt64)?; @@ -6198,7 +6228,38 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { } } } - } else { + } else if opcode == YARVINSN_getblockparamproxy || opcode == YARVINSN_trace_getblockparamproxy { + if get_option!(stats) { + let iseq_insn_idx = exit_state.insn_idx; + if let Some([block_handler_distribution]) = profiles.payload.profile.get_operand_types(iseq_insn_idx) { + let summary = TypeDistributionSummary::new(block_handler_distribution); + + if summary.is_monomorphic() { + let obj = summary.bucket(0).class(); + if unsafe { rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1} { + fun.push_insn(block, Insn::IncrCounter(Counter::getblockparamproxy_handler_iseq)); + } else if unsafe { rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1} { + fun.push_insn(block, Insn::IncrCounter(Counter::getblockparamproxy_handler_ifunc)); + } + else if obj.nil_p() { + fun.push_insn(block, Insn::IncrCounter(Counter::getblockparamproxy_handler_nil)); + } + else if obj.symbol_p() { + fun.push_insn(block, Insn::IncrCounter(Counter::getblockparamproxy_handler_symbol)); + } else if unsafe { rb_obj_is_proc(obj).test() } { + fun.push_insn(block, Insn::IncrCounter(Counter::getblockparamproxy_handler_proc)); + } + } else if summary.is_polymorphic() || summary.is_skewed_polymorphic() { + fun.push_insn(block, Insn::IncrCounter(Counter::getblockparamproxy_handler_polymorphic)); + } else if summary.is_megamorphic() || summary.is_skewed_megamorphic() { + fun.push_insn(block, Insn::IncrCounter(Counter::getblockparamproxy_handler_megamorphic)); + } + } else { + fun.push_insn(block, Insn::IncrCounter(Counter::getblockparamproxy_handler_no_profiles)); + } + } + } + else { profiles.profile_stack(&exit_state); } @@ -6593,9 +6654,39 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { } YARVINSN_getblockparamproxy => { let level = get_arg(pc, 1).as_u32(); - fun.push_insn(block, Insn::GuardBlockParamProxy { level, state: exit_id }); - // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing - state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) })); + + let profiled_block_type = if let Some([block_handler_distribution]) = profiles.payload.profile.get_operand_types(exit_state.insn_idx) { + let summary = TypeDistributionSummary::new(block_handler_distribution); + summary.is_monomorphic().then_some(summary.bucket(0).class()) + } else { + None + }; + + let ep = fun.push_insn(block, Insn::GetEP { level }); + let flags = fun.push_insn(block, Insn::LoadField { recv: ep, id: ID!(_env_data_index_flags), offset: SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32), return_type: types::CInt64 }); + fun.push_insn(block, Insn::GuardNoBitsSet { val: flags, mask: Const::CUInt64(VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()), reason: SideExitReason::BlockParamProxyModified, state: exit_id }); + + let block_handler = fun.push_insn(block, Insn::LoadField { recv: ep, id: ID!(_env_data_index_specval), offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, return_type: types::CInt64 }); + + match profiled_block_type { + Some(ty) if ty.nil_p() => { + fun.push_insn(block, Insn::GuardBitEquals { val: block_handler, expected: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), reason: SideExitReason::BlockParamProxyNotNil, state: exit_id }); + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) })); + } + _ => { + // This handles two cases which are nearly identical + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); + + // Bail out if the block handler is neither ISEQ nor ifunc + fun.push_insn(block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), reason: SideExitReason::BlockParamProxyNotIseqOrIfunc, state: exit_id }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) })); + } + } } YARVINSN_getblockparam => { fn new_branch_block( diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 70afd54022..c1059094ac 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -3875,7 +3875,7 @@ mod hir_opt_tests { eval(" def test(&block) = tap(&block) "); - assert_snapshot!(hir_string("test"), @r" + assert_snapshot!(hir_string("test"), @" fn test@<compiled>:2: bb0(): EntryPoint interpreter @@ -3886,11 +3886,15 @@ mod hir_opt_tests { EntryPoint JIT(0) Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): - GuardBlockParamProxy l0 - v15:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) - v17:BasicObject = Send v8, 0x1008, :tap, v15 # SendFallbackReason: Uncategorized(send) + v14:CPtr = GetEP 0 + v15:CInt64 = LoadField v14, :_env_data_index_flags@0x1000 + v16:CInt64 = GuardBitNotSet v15, CUInt64(512) + v17:CInt64 = LoadField v14, :_env_data_index_specval@0x1001 + v18:CInt64 = GuardBitSet v17, CUInt64(1) + v19:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1008)) + v21:BasicObject = Send v8, 0x1010, :tap, v19 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v17 + Return v21 "); } @@ -6520,9 +6524,40 @@ mod hir_opt_tests { fn test_do_not_optimize_send_with_block_forwarding() { eval(r#" def test(&block) = [].map(&block) + test { |x| x }; test { |x| x } + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :block, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:ArrayExact = NewArray + v15:CPtr = GetEP 0 + v16:CInt64 = LoadField v15, :_env_data_index_flags@0x1000 + v17:CInt64 = GuardBitNotSet v16, CUInt64(512) + v18:CInt64 = LoadField v15, :_env_data_index_specval@0x1001 + v19:CInt64 = GuardBitSet v18, CUInt64(1) + v20:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1008)) + IncrCounter complex_arg_pass_caller_blockarg + v22:BasicObject = Send v13, 0x1010, :map, v20 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_replace_block_param_proxy_with_nil() { + eval(r#" + def test(&block) = [].map(&block) test; test "#); - assert_snapshot!(hir_string("test"), @r" + assert_snapshot!(hir_string("test"), @" fn test@<compiled>:2: bb0(): EntryPoint interpreter @@ -6534,12 +6569,50 @@ mod hir_opt_tests { Jump bb2(v5, v6) bb2(v8:BasicObject, v9:BasicObject): v13:ArrayExact = NewArray - GuardBlockParamProxy l0 - v16:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + v15:CPtr = GetEP 0 + v16:CInt64 = LoadField v15, :_env_data_index_flags@0x1000 + v17:CInt64 = GuardBitNotSet v16, CUInt64(512) + v18:CInt64 = LoadField v15, :_env_data_index_specval@0x1001 + v19:CInt64[0] = GuardBitEquals v18, CInt64(0) + v20:NilClass = Const Value(nil) IncrCounter complex_arg_pass_caller_blockarg - v18:BasicObject = Send v13, 0x1008, :map, v16 # SendFallbackReason: Complex argument passing + v22:BasicObject = Send v13, 0x1008, :map, v20 # SendFallbackReason: Complex argument passing CheckInterrupts - Return v18 + Return v22 + "); + } + + #[test] + fn test_replace_block_param_proxy_with_nil_nested() { + eval(r#" + def test(&block) + proc do + [].map(&block) + end + end + test; test + "#); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + v12:CPtr = GetEP 1 + v13:CInt64 = LoadField v12, :_env_data_index_flags@0x1000 + v14:CInt64 = GuardBitNotSet v13, CUInt64(512) + v15:CInt64 = LoadField v12, :_env_data_index_specval@0x1001 + v16:CInt64 = GuardBitSet v15, CUInt64(1) + v17:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1008)) + IncrCounter complex_arg_pass_caller_blockarg + v19:BasicObject = Send v10, 0x1010, :map, v17 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v19 "); } diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index e0b0129ea1..dbab964976 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -2038,7 +2038,7 @@ pub mod hir_build_tests { eval(" def test(a, ...) = foo(a, ...) "); - assert_snapshot!(hir_string("test"), @r" + assert_snapshot!(hir_string("test"), @" fn test@<compiled>:2: bb0(): EntryPoint interpreter @@ -2056,8 +2056,12 @@ pub mod hir_build_tests { bb2(v16:BasicObject, v17:BasicObject, v18:ArrayExact, v19:BasicObject, v20:BasicObject, v21:NilClass): v28:ArrayExact = ToArray v18 PatchPoint NoEPEscape(test) - GuardBlockParamProxy l0 - v34:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + v33:CPtr = GetEP 0 + v34:CInt64 = LoadField v33, :_env_data_index_flags@0x1000 + v35:CInt64 = GuardBitNotSet v34, CUInt64(512) + v36:CInt64 = LoadField v33, :_env_data_index_specval@0x1001 + v37:CInt64 = GuardBitSet v36, CUInt64(1) + v38:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1008)) SideExit UnhandledYARVInsn(splatkw) "); } @@ -3409,7 +3413,7 @@ pub mod hir_build_tests { let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Dir", "open")); assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate), "iseq Dir.open does not contain invokebuiltin"); let function = iseq_to_hir(iseq).unwrap(); - assert_snapshot!(hir_string_function(&function), @r" + assert_snapshot!(hir_string_function(&function), @" fn open@<internal:dir>: bb0(): EntryPoint interpreter @@ -3428,20 +3432,24 @@ pub mod hir_build_tests { bb2(v16:BasicObject, v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:NilClass): v25:BasicObject = InvokeBuiltin dir_s_open, v16, v17, v18 PatchPoint NoEPEscape(open) - GuardBlockParamProxy l0 - v32:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + v31:CPtr = GetEP 0 + v32:CInt64 = LoadField v31, :_env_data_index_flags@0x1000 + v33:CInt64 = GuardBitNotSet v32, CUInt64(512) + v34:CInt64 = LoadField v31, :_env_data_index_specval@0x1001 + v35:CInt64 = GuardBitSet v34, CUInt64(1) + v36:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1008)) + CheckInterrupts + v39:CBool[true] = Test v36 + v40 = RefineType v36, Falsy + IfFalse v39, bb3(v16, v17, v18, v19, v20, v25) + v42:HeapObject[BlockParamProxy] = RefineType v36, Truthy + v46:BasicObject = InvokeBlock, v25 # SendFallbackReason: Uncategorized(invokeblock) + v49:BasicObject = InvokeBuiltin dir_s_close, v16, v25 CheckInterrupts - v35:CBool[true] = Test v32 - v36 = RefineType v32, Falsy - IfFalse v35, bb3(v16, v17, v18, v19, v20, v25) - v38:HeapObject[BlockParamProxy] = RefineType v32, Truthy - v42:BasicObject = InvokeBlock, v25 # SendFallbackReason: Uncategorized(invokeblock) - v45:BasicObject = InvokeBuiltin dir_s_close, v16, v25 - CheckInterrupts - Return v42 - bb3(v51, v52, v53, v54, v55, v56): + Return v46 + bb3(v55, v56, v57, v58, v59, v60): CheckInterrupts - Return v56 + Return v60 "); } diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs index c1feb75952..ad6da06c71 100644 --- a/zjit/src/profile.rs +++ b/zjit/src/profile.rs @@ -91,6 +91,7 @@ fn profile_insn(bare_opcode: ruby_vminsn_type, ec: EcPtr) { YARVINSN_opt_size => profile_operands(profiler, profile, 1), YARVINSN_opt_succ => profile_operands(profiler, profile, 1), YARVINSN_invokeblock => profile_block_handler(profiler, profile), + YARVINSN_getblockparamproxy => profile_getblockparamproxy(profiler, profile), YARVINSN_invokesuper => profile_invokesuper(profiler, profile), YARVINSN_opt_send_without_block | YARVINSN_send => { let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); @@ -155,6 +156,22 @@ fn profile_block_handler(profiler: &mut Profiler, profile: &mut IseqProfile) { types[0].observe(ty); } +fn profile_getblockparamproxy(profiler: &mut Profiler, profile: &mut IseqProfile) { + let types = &mut profile.opnd_types[profiler.insn_idx]; + if types.is_empty() { + types.resize(1, TypeDistribution::new()); + } + + let level = profiler.insn_opnd(1).as_u32(); + let ep = unsafe { get_cfp_ep_level(profiler.cfp, level) }; + let block_handler = unsafe { *ep.offset(VM_ENV_DATA_INDEX_SPECVAL as isize) }; + let untagged = unsafe { rb_vm_untag_block_handler(block_handler) }; + + let ty = ProfiledType::object(untagged); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + types[0].observe(ty); +} + fn profile_invokesuper(profiler: &mut Profiler, profile: &mut IseqProfile) { let cme = unsafe { rb_vm_frame_method_entry(profiler.cfp) }; let cme_value = VALUE(cme as usize); // CME is a T_IMEMO, which is a VALUE diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index bb11b96dd9..367a19fc32 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -210,6 +210,7 @@ make_counters! { exit_stackoverflow, exit_block_param_proxy_modified, exit_block_param_proxy_not_iseq_or_ifunc, + exit_block_param_proxy_not_nil, exit_block_param_wb_required, exit_too_many_keyword_parameters, } @@ -422,6 +423,15 @@ make_counters! { invokeblock_handler_polymorphic, invokeblock_handler_megamorphic, invokeblock_handler_no_profiles, + + getblockparamproxy_handler_iseq, + getblockparamproxy_handler_ifunc, + getblockparamproxy_handler_symbol, + getblockparamproxy_handler_proc, + getblockparamproxy_handler_nil, + getblockparamproxy_handler_polymorphic, + getblockparamproxy_handler_megamorphic, + getblockparamproxy_handler_no_profiles, } /// Increase a counter by a specified amount @@ -558,6 +568,7 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { StackOverflow => exit_stackoverflow, BlockParamProxyModified => exit_block_param_proxy_modified, BlockParamProxyNotIseqOrIfunc => exit_block_param_proxy_not_iseq_or_ifunc, + BlockParamProxyNotNil => exit_block_param_proxy_not_nil, BlockParamWbRequired => exit_block_param_wb_required, TooManyKeywordParameters => exit_too_many_keyword_parameters, PatchPoint(Invariant::BOPRedefined { .. }) |
